From b3859cc0bddb2f21595254a47a6d31bc58a94387 Mon Sep 17 00:00:00 2001 From: Arthur Galuza Date: Mon, 4 Oct 2021 17:57:29 +0300 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20Source=20Iterable:=20add=20new?= =?UTF-8?q?=20streams=20(#5915)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add new streams * Upd requirements versions * Upd docs * Remove tests for the templates stream * Upd csv field parsing * Fix file permissions * Set dependency version * Refactor * Merge * Upd licence * Add bulk metrics retrieving * Actualize schema --- .../connectors/source-iterable/Dockerfile | 2 +- .../acceptance-test-config.yml | 24 + .../source-iterable/acceptance-test-docker.sh | 15 + .../connectors/source-iterable/build.gradle | 11 +- .../integration_tests/__init__.py | 0 .../integration_tests/abnormal_state.json | 26 + .../integration_tests/acceptance.py | 13 + .../integration_tests/catalog.json | 186 +++ .../integration_tests/configured_catalog.json | 175 +++ .../integration_tests/invalid_config.json | 4 + .../source-iterable/requirements.txt | 2 + .../sample_files/configured_catalog.json | 1258 ----------------- .../sample_files/sample_config.json | 2 +- .../source-iterable/sample_files/state.json | 26 - .../connectors/source-iterable/setup.py | 8 +- .../source-iterable/source_iterable/api.py | 161 ++- .../schemas/campaigns_metrics.json | 8 + .../source_iterable/schemas/events.json | 8 + .../source_iterable/schemas/metadata.json | 62 +- .../source-iterable/source_iterable/source.py | 4 + docs/integrations/sources/iterable.md | 5 +- 21 files changed, 623 insertions(+), 1377 deletions(-) create mode 100644 airbyte-integrations/connectors/source-iterable/acceptance-test-config.yml create mode 100755 airbyte-integrations/connectors/source-iterable/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-iterable/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-iterable/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-iterable/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-iterable/integration_tests/catalog.json create mode 100644 airbyte-integrations/connectors/source-iterable/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-iterable/integration_tests/invalid_config.json delete mode 100644 airbyte-integrations/connectors/source-iterable/sample_files/configured_catalog.json delete mode 100644 airbyte-integrations/connectors/source-iterable/sample_files/state.json mode change 100644 => 100755 airbyte-integrations/connectors/source-iterable/source_iterable/api.py create mode 100644 airbyte-integrations/connectors/source-iterable/source_iterable/schemas/campaigns_metrics.json create mode 100644 airbyte-integrations/connectors/source-iterable/source_iterable/schemas/events.json diff --git a/airbyte-integrations/connectors/source-iterable/Dockerfile b/airbyte-integrations/connectors/source-iterable/Dockerfile index 5f95202fb058..453697195e7b 100644 --- a/airbyte-integrations/connectors/source-iterable/Dockerfile +++ b/airbyte-integrations/connectors/source-iterable/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/source-iterable diff --git a/airbyte-integrations/connectors/source-iterable/acceptance-test-config.yml b/airbyte-integrations/connectors/source-iterable/acceptance-test-config.yml new file mode 100644 index 000000000000..587d1172565a --- /dev/null +++ b/airbyte-integrations/connectors/source-iterable/acceptance-test-config.yml @@ -0,0 +1,24 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/contributing-to-airbyte/building-new-connector/source-acceptance-tests.md) +# for more information about how to configure these tests +connector_image: airbyte/source-iterable:dev +tests: + spec: + - spec_path: "source_iterable/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/catalog.json" + empty_streams: ['email_send_skip', 'email_complaint'] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/catalog.json" + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" diff --git a/airbyte-integrations/connectors/source-iterable/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-iterable/acceptance-test-docker.sh new file mode 100755 index 000000000000..c522eebbd94e --- /dev/null +++ b/airbyte-integrations/connectors/source-iterable/acceptance-test-docker.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input diff --git a/airbyte-integrations/connectors/source-iterable/build.gradle b/airbyte-integrations/connectors/source-iterable/build.gradle index f55ea0aca67f..e95e96c3c030 100644 --- a/airbyte-integrations/connectors/source-iterable/build.gradle +++ b/airbyte-integrations/connectors/source-iterable/build.gradle @@ -1,20 +1,13 @@ plugins { id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-standard-source-test-file' + id 'airbyte-source-acceptance-test' } airbytePython { moduleDirectory 'source_iterable' } -airbyteStandardSourceTestFile { - specPath = "source_iterable/spec.json" - configPath = "secrets/config.json" - configuredCatalogPath = "sample_files/configured_catalog.json" -} - - dependencies { - implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-iterable/integration_tests/__init__.py b/airbyte-integrations/connectors/source-iterable/integration_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-iterable/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-iterable/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..0fd5d83fe5dd --- /dev/null +++ b/airbyte-integrations/connectors/source-iterable/integration_tests/abnormal_state.json @@ -0,0 +1,26 @@ +{ + "users": { + "profileUpdatedAt": "2121-04-14T17:00:41+00:00" + }, + "email_unsubscribe": { + "createdAt": "2121-04-14T17:00:44+00:00" + }, + "email_subscribe": { + "createdAt": "2121-04-14T16:52:45+00:00" + }, + "email_send": { + "createdAt": "2121-04-14T16:25:56+00:00" + }, + "email_open": { + "createdAt": "2121-04-14T17:00:11+00:00" + }, + "email_click": { + "createdAt": "2121-04-14T16:55:14+00:00" + }, + "email_bounce": { + "createdAt": "2121-04-14T16:29:39+00:00" + }, + "templates": { + "createdAt": "2121-04-14T16:23:30.700000+00:00" + } +} diff --git a/airbyte-integrations/connectors/source-iterable/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-iterable/integration_tests/acceptance.py new file mode 100644 index 000000000000..a294f4dcc5b7 --- /dev/null +++ b/airbyte-integrations/connectors/source-iterable/integration_tests/acceptance.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-iterable/integration_tests/catalog.json b/airbyte-integrations/connectors/source-iterable/integration_tests/catalog.json new file mode 100644 index 000000000000..02f67b017754 --- /dev/null +++ b/airbyte-integrations/connectors/source-iterable/integration_tests/catalog.json @@ -0,0 +1,186 @@ +{ + "streams": [ + { + "stream": { + "name": "campaigns", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "campaigns_metrics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "channels", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "email_bounce", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "email_click", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "email_complaint", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "email_open", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "email_send", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "email_send_skip", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "email_subscribe", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "email_unsubscribe", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "events", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "lists", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "list_users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "message_types", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "metadata", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "templates", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["profileUpdatedAt"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-iterable/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-iterable/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..a6392effd040 --- /dev/null +++ b/airbyte-integrations/connectors/source-iterable/integration_tests/configured_catalog.json @@ -0,0 +1,175 @@ +{ + "streams": [ + { + "stream": { + "name": "campaigns", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "campaigns_metrics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "channels", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "email_bounce", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "email_click", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "email_complaint", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "email_open", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "email_send", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "email_send_skip", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "email_subscribe", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "email_unsubscribe", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "events", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "lists", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "list_users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "message_types", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "metadata", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["profileUpdatedAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-iterable/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-iterable/integration_tests/invalid_config.json new file mode 100644 index 000000000000..33e5a4a510ec --- /dev/null +++ b/airbyte-integrations/connectors/source-iterable/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "test-api-key", + "start_date": "2020-12-12T00:00:00Z" +} diff --git a/airbyte-integrations/connectors/source-iterable/requirements.txt b/airbyte-integrations/connectors/source-iterable/requirements.txt index d6e1198b1ab1..7be17a56d745 100644 --- a/airbyte-integrations/connectors/source-iterable/requirements.txt +++ b/airbyte-integrations/connectors/source-iterable/requirements.txt @@ -1 +1,3 @@ +# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. +-e ../../bases/source-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-iterable/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-iterable/sample_files/configured_catalog.json deleted file mode 100644 index 2b59c1b02dcf..000000000000 --- a/airbyte-integrations/connectors/source-iterable/sample_files/configured_catalog.json +++ /dev/null @@ -1,1258 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "campaigns", - "json_schema": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "updatedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "startAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "endedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "name": { - "type": ["null", "string"] - }, - "templateId": { - "type": ["null", "integer"] - }, - "messageMedium": { - "type": ["null", "string"] - }, - "createdByUserId": { - "type": ["null", "string"] - }, - "updatedByUserId": { - "type": ["null", "string"] - }, - "campaignState": { - "type": ["null", "string"] - }, - "listIds": { - "type": ["null", "array"], - "items": {} - }, - "suppressionListIds": { - "type": ["null", "array"], - "items": {} - }, - "sendSize": { - "type": ["null", "number"] - }, - "recurringCampaignId": { - "type": ["null", "number"] - }, - "workflowId": { - "type": ["null", "number"] - }, - "labels": { - "type": ["null", "array"], - "items": {} - }, - "type": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "channels", - "json_schema": { - "properties": { - "id": { - "type": ["null", "number"] - }, - "name": { - "type": ["null", "string"] - }, - "channelType": { - "type": ["null", "string"] - }, - "messageMedium": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "email_bounce", - "json_schema": { - "properties": { - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "campaignId": { - "type": ["null", "integer"] - }, - "itblInternal": { - "type": ["null", "object"], - "properties": { - "documentCreatedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "documentUpdatedAt": { - "type": ["null", "string"], - "format": "date-time" - } - } - }, - "messageId": { - "type": ["null", "string"] - }, - "templateId": { - "type": ["null", "integer"] - }, - "email": { - "type": ["null", "string"] - }, - "recipientState": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["createdAt"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "email_click", - "json_schema": { - "properties": { - "country": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "campaignId": { - "type": ["null", "integer"] - }, - "itblInternal": { - "type": ["null", "object"], - "properties": { - "documentCreatedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "documentUpdatedAt": { - "type": ["null", "string"], - "format": "date-time" - } - } - }, - "ip": { - "type": ["null", "string"] - }, - "contentId": { - "type": ["null", "integer"] - }, - "userAgentDevice": { - "type": ["null", "string"] - }, - "messageId": { - "type": ["null", "string"] - }, - "hrefIndex": { - "type": ["null", "integer"] - }, - "userAgent": { - "type": ["null", "string"] - }, - "templateId": { - "type": ["null", "integer"] - }, - "url": { - "type": ["null", "string"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "region": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["createdAt"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "email_complaint", - "json_schema": { - "properties": { - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "campaignId": { - "type": ["null", "integer"] - }, - "itblInternal": { - "type": ["null", "object"], - "properties": { - "documentCreatedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "documentUpdatedAt": { - "type": ["null", "string"], - "format": "date-time" - } - } - }, - "messageId": { - "type": ["null", "string"] - }, - "templateId": { - "type": ["null", "integer"] - }, - "email": { - "type": ["null", "string"] - }, - "recipientState": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["createdAt"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "email_open", - "json_schema": { - "properties": { - "country": { - "type": ["null", "string"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "city": { - "type": ["null", "string"] - }, - "campaignId": { - "type": ["null", "integer"] - }, - "itblInternal": { - "type": ["null", "object"], - "properties": { - "documentCreatedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "documentUpdatedAt": { - "type": ["null", "string"], - "format": "date-time" - } - } - }, - "ip": { - "type": ["null", "string"] - }, - "userAgentDevice": { - "type": ["null", "string"] - }, - "messageId": { - "type": ["null", "string"] - }, - "userAgent": { - "type": ["null", "string"] - }, - "templateId": { - "type": ["null", "integer"] - }, - "region": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["createdAt"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "email_send", - "json_schema": { - "properties": { - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "campaignId": { - "type": ["null", "integer"] - }, - "itblInternal": { - "type": ["null", "object"], - "properties": { - "documentCreatedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "documentUpdatedAt": { - "type": ["null", "string"], - "format": "date-time" - } - } - }, - "messageTypeId": { - "type": ["null", "integer"] - }, - "transactionalData": { - "type": ["null", "object"], - "properties": { - "inventory": { - "type": ["null", "integer"] - }, - "eventName": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "sku": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "price": { - "type": ["null", "integer"] - }, - "product_type": { - "type": ["null", "string"] - }, - "compare_at_price": { - "type": ["null", "number"] - }, - "id": { - "type": ["null", "string"] - }, - "templateId": { - "type": ["null", "integer"] - }, - "product_id": { - "type": ["null", "string"] - }, - "categories": { - "type": ["null", "array"], - "items": {} - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "campaignId": { - "type": ["null", "integer"] - }, - "vendor": { - "type": ["null", "string"] - }, - "eventUpdatedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "discount": { - "type": ["null", "integer"] - }, - "imageUrl": { - "type": ["null", "string"] - }, - "itblInternal": { - "type": ["null", "object"], - "properties": { - "documentCreatedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "documentUpdatedAt": { - "type": ["null", "string"], - "format": "date-time" - } - } - }, - "handle": { - "type": ["null", "string"] - } - } - }, - "contentId": { - "type": ["null", "integer"] - }, - "messageId": { - "type": ["null", "string"] - }, - "messageBusId": { - "type": ["null", "string"] - }, - "templateId": { - "type": ["null", "integer"] - }, - "email": { - "type": ["null", "string"] - }, - "channelId": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["createdAt"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "email_send_skip", - "json_schema": { - "properties": { - "reason": { - "type": ["null", "string"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "campaignId": { - "type": ["null", "integer"] - }, - "itblInternal": { - "type": ["null", "object"], - "properties": { - "documentCreatedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "documentUpdatedAt": { - "type": ["null", "string"], - "format": "date-time" - } - } - }, - "messageTypeId": { - "type": ["null", "integer"] - }, - "transactionalData": { - "type": ["null", "object"], - "properties": { - "inventory": { - "type": ["null", "integer"] - }, - "eventName": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "sku": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "price": { - "type": ["null", "integer"] - }, - "product_type": { - "type": ["null", "string"] - }, - "compare_at_price": { - "type": ["null", "number"] - }, - "id": { - "type": ["null", "string"] - }, - "templateId": { - "type": ["null", "integer"] - }, - "product_id": { - "type": ["null", "string"] - }, - "categories": { - "type": ["null", "array"], - "items": {} - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "campaignId": { - "type": ["null", "integer"] - }, - "vendor": { - "type": ["null", "string"] - }, - "eventUpdatedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "discount": { - "type": ["null", "integer"] - }, - "imageUrl": { - "type": ["null", "string"] - }, - "itblInternal": { - "type": ["null", "object"], - "properties": { - "documentCreatedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "documentUpdatedAt": { - "type": ["null", "string"], - "format": "date-time" - } - } - }, - "handle": { - "type": ["null", "string"] - } - } - }, - "contentId": { - "type": ["null", "integer"] - }, - "messageId": { - "type": ["null", "string"] - }, - "templateId": { - "type": ["null", "integer"] - }, - "email": { - "type": ["null", "string"] - }, - "channelId": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["createdAt"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "email_subscribe", - "json_schema": { - "properties": { - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "signupSource": { - "type": ["null", "string"] - }, - "emailListIds": { - "type": ["null", "array"], - "items": {} - }, - "itblInternal": { - "type": ["null", "object"], - "properties": { - "documentCreatedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "documentUpdatedAt": { - "type": ["null", "string"], - "format": "date-time" - } - } - }, - "emailListId": { - "type": ["null", "integer"] - }, - "email": { - "type": ["null", "string"] - }, - "profileUpdatedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "workflowId": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["createdAt"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "email_unsubscribe", - "json_schema": { - "properties": { - "unsubSource": { - "type": ["null", "string"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "campaignId": { - "type": ["null", "integer"] - }, - "itblInternal": { - "type": ["null", "object"], - "properties": { - "documentCreatedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "documentUpdatedAt": { - "type": ["null", "string"], - "format": "date-time" - } - } - }, - "emailListId": { - "type": ["null", "integer"] - }, - "emailListIds": { - "type": ["null", "array"], - "items": {} - }, - "workflowId": { - "type": ["null", "integer"] - }, - "messageId": { - "type": ["null", "string"] - }, - "templateId": { - "type": ["null", "integer"] - }, - "channelIds": { - "type": ["null", "array"], - "items": {} - }, - "email": { - "type": ["null", "string"] - }, - "channelId": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["createdAt"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "lists", - "json_schema": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "listType": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "list_users", - "json_schema": { - "properties": { - "email": { - "type": ["null", "string"] - }, - "listId": { - "type": ["null", "integer"] - } - }, - "type": ["null", "object"] - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "message_types", - "json_schema": { - "properties": { - "id": { - "type": ["null", "number"] - }, - "name": { - "type": ["null", "string"] - }, - "channelId": { - "type": ["null", "number"] - } - }, - "type": ["null", "object"] - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "metadata", - "json_schema": { - "properties": { - "table": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "size": { - "type": ["null", "integer"] - }, - "lastModified": { - "type": ["null", "string"], - "format": "date-time" - }, - "value": { - "type": ["null", "object"], - "properties": { - "inventory": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "sku": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "price": { - "type": ["null", "integer"] - }, - "product_type": { - "type": ["null", "string"] - }, - "compare_at_price": { - "type": ["null", "number"] - }, - "id": { - "type": ["null", "string"] - }, - "product_id": { - "type": ["null", "string"] - }, - "categories": { - "type": ["null", "array"], - "items": {} - }, - "vendor": { - "type": ["null", "string"] - }, - "discount": { - "type": ["null", "integer"] - }, - "imageUrl": { - "type": ["null", "string"] - }, - "handle": { - "type": ["null", "string"] - } - } - } - }, - "type": ["null", "object"] - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "templates", - "json_schema": { - "properties": { - "templateId": { - "type": ["null", "number"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "updatedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "name": { - "type": ["null", "string"] - }, - "creatorUserId": { - "type": ["null", "string"] - }, - "messageTypeId": { - "type": ["null", "number"] - }, - "campaignId": { - "type": ["null", "number"] - }, - "clientTemplateId": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["createdAt"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "users", - "json_schema": { - "properties": { - "country": { - "type": ["null", "string"] - }, - "firstOrderDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "addresses": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "first_name": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "zip": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "address1": { - "type": ["null", "string"] - }, - "address2": { - "type": ["null", "string"] - }, - "company": { - "type": ["null", "string"] - }, - "country_code": { - "type": ["null", "string"] - }, - "default": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "province": { - "type": ["null", "string"] - }, - "province_code": { - "type": ["null", "string"] - }, - "country_name": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - } - } - } - }, - "emailAcquiredDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "emailSegmentStatus": { - "type": ["null", "string"] - }, - "admin_graphql_api_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "mostRecentEmailList": { - "type": ["null", "string"] - }, - "mostRecentEmailSegment": { - "type": ["null", "string"] - }, - "aov": { - "type": ["null", "number"] - }, - "firstCampaign": { - "type": ["null", "string"] - }, - "thirdMostRecentOrderDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "firstPurchaseDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "firstMedium": { - "type": ["null", "string"] - }, - "default_address": { - "type": ["null", "object"], - "properties": { - "first_name": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "zip": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "address2": { - "type": ["null", "string"] - }, - "company": { - "type": ["null", "string"] - }, - "country_code": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "address1": { - "type": ["null", "string"] - }, - "default": { - "type": ["null", "boolean"] - }, - "province": { - "type": ["null", "string"] - }, - "province_code": { - "type": ["null", "string"] - }, - "country_name": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - } - } - }, - "emailListIds": { - "type": ["null", "array"], - "items": {} - }, - "accepts_marketing": { - "type": ["null", "boolean"] - }, - "secondMostRecentOrderDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "state": { - "type": ["null", "string"] - }, - "mostRecentCampaign": { - "type": ["null", "string"] - }, - "zip": { - "type": ["null", "string"] - }, - "total_spent": { - "type": ["null", "number"] - }, - "mostRecentOrderDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "last_order_id": { - "type": ["null", "string"] - }, - "tax_exempt": { - "type": ["null", "boolean"] - }, - "mostRecentSource": { - "type": ["null", "string"] - }, - "twelveMonthLtr": { - "type": ["null", "integer"] - }, - "verified_email": { - "type": ["null", "boolean"] - }, - "mostRecentMedium": { - "type": ["null", "string"] - }, - "orders_count": { - "type": ["null", "integer"] - }, - "firstName": { - "type": ["null", "string"] - }, - "lastInteractionTs": { - "type": ["null", "string"], - "format": "date-time" - }, - "boughtSas": { - "type": ["null", "boolean"] - }, - "secondMostRecentOrderCards": { - "type": ["null", "array"], - "items": {} - }, - "unsubscribedChannelIds": { - "type": ["null", "array"], - "items": {} - }, - "lastName": { - "type": ["null", "string"] - }, - "last_order_name": { - "type": ["null", "string"] - }, - "secondOrderDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "hasAccount": { - "type": ["null", "boolean"] - }, - "city": { - "type": ["null", "string"] - }, - "mostRecentOrderCards": { - "type": ["null", "array"], - "items": {} - }, - "itblInternal": { - "type": ["null", "object"], - "properties": { - "emailDomain": { - "type": ["null", "string"] - }, - "documentUpdatedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "documentCreatedAt": { - "type": ["null", "string"], - "format": "date-time" - } - } - }, - "hasReminder": { - "type": ["null", "boolean"] - }, - "thirdOrderDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "subscribedMessageTypeIds": { - "type": ["null", "array"], - "items": {} - }, - "firstSource": { - "type": ["null", "string"] - }, - "unsubscribedMessageTypeIds": { - "type": ["null", "array"], - "items": {} - }, - "first_name": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "thirdMostRecentOrderCards": { - "type": ["null", "array"], - "items": {} - }, - "profileUpdatedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "signupDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "businessLines": { - "type": ["null", "array"], - "items": {} - }, - "secondOrderCards": { - "type": ["null", "array"], - "items": {} - }, - "address1": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "ltr": { - "type": ["null", "integer"] - }, - "userId": { - "type": ["null", "string"] - }, - "shopify_created_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "signupSource": { - "type": ["null", "string"] - }, - "thirdOrderCards": { - "type": ["null", "array"], - "items": {} - }, - "firstOrderCards": { - "type": ["null", "array"], - "items": {} - }, - "totalOrders": { - "type": ["null", "integer"] - }, - "shopify_updated_at": { - "type": ["null", "string"], - "format": "date-time" - } - }, - "type": ["null", "object"] - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["profileUpdatedAt"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - } - ] -} diff --git a/airbyte-integrations/connectors/source-iterable/sample_files/sample_config.json b/airbyte-integrations/connectors/source-iterable/sample_files/sample_config.json index ea49b1ae643a..fb46b1ec2582 100644 --- a/airbyte-integrations/connectors/source-iterable/sample_files/sample_config.json +++ b/airbyte-integrations/connectors/source-iterable/sample_files/sample_config.json @@ -1,4 +1,4 @@ { - "api_key": ">", + "api_key": "", "start_date": "2021-04-01T00:00:00Z" } diff --git a/airbyte-integrations/connectors/source-iterable/sample_files/state.json b/airbyte-integrations/connectors/source-iterable/sample_files/state.json deleted file mode 100644 index eeb8474cdf8a..000000000000 --- a/airbyte-integrations/connectors/source-iterable/sample_files/state.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "users": { - "profileUpdatedAt": "2021-04-14T17:00:41+00:00" - }, - "email_unsubscribe": { - "createdAt": "2021-04-14T17:00:44+00:00" - }, - "email_subscribe": { - "createdAt": "2021-04-14T16:52:45+00:00" - }, - "email_send": { - "createdAt": "2021-04-14T16:25:56+00:00" - }, - "email_open": { - "createdAt": "2021-04-14T17:00:11+00:00" - }, - "email_click": { - "createdAt": "2021-04-14T16:55:14+00:00" - }, - "email_bounce": { - "createdAt": "2021-04-14T16:29:39+00:00" - }, - "templates": { - "createdAt": "2021-04-14T16:23:30.700000+00:00" - } -} diff --git a/airbyte-integrations/connectors/source-iterable/setup.py b/airbyte-integrations/connectors/source-iterable/setup.py index ee26ba9234ff..90f33d5300e3 100644 --- a/airbyte-integrations/connectors/source-iterable/setup.py +++ b/airbyte-integrations/connectors/source-iterable/setup.py @@ -6,12 +6,12 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk==0.1.3", - "pendulum==1.2.0", - "requests==2.25.1", + "airbyte-cdk~=0.1", + "pendulum~=1.2", + "requests~=2.25", ] -TEST_REQUIREMENTS = ["pytest==6.1.2"] +TEST_REQUIREMENTS = ["pytest~=6.1"] setup( diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/api.py b/airbyte-integrations/connectors/source-iterable/source_iterable/api.py old mode 100644 new mode 100755 index 2a07229c0c52..126c8837924b --- a/airbyte-integrations/connectors/source-iterable/source_iterable/api.py +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/api.py @@ -2,23 +2,29 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # - +import csv import json import urllib.parse as urlparse from abc import ABC, abstractmethod -from typing import Any, Iterable, Mapping, MutableMapping, Optional, Union +from io import StringIO +from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Union import pendulum import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream +EVENT_ROWS_LIMIT = 200 +CAMPAIGNS_PER_REQUEST = 20 + class IterableStream(HttpStream, ABC): - url_base = "https://api.iterable.com/api/" # Hardcode the value because it is not returned from the API BACKOFF_TIME_CONSTANT = 10.0 + # define date-time fields with potential wrong format + + url_base = "https://api.iterable.com/api/" primary_key = "id" def __init__(self, api_key, **kwargs): @@ -46,16 +52,24 @@ def request_params(self, **kwargs) -> MutableMapping[str, Any]: def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: response_json = response.json() - yield from response_json.get(self.data_field, []) + records = response_json.get(self.data_field, []) + + for record in records: + yield record class IterableExportStream(IterableStream, ABC): + + cursor_field = "createdAt" + primary_key = None + def __init__(self, start_date, **kwargs): super().__init__(**kwargs) self._start_date = pendulum.parse(start_date) self.stream_params = {"dataTypeName": self.data_field} - cursor_field = "createdAt" + def path(self, **kwargs) -> str: + return "/export/data.json" @staticmethod def _field_to_datetime(value: Union[int, str]) -> pendulum.datetime: @@ -72,10 +86,14 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object and returning an updated state object. """ - latest_benchmark = self._field_to_datetime(latest_record[self.cursor_field]) + latest_benchmark = latest_record[self.cursor_field] if current_stream_state.get(self.cursor_field): - return {self.cursor_field: str(max(latest_benchmark, self._field_to_datetime(current_stream_state[self.cursor_field])))} - return {self.cursor_field: str(latest_benchmark)} + return { + self.cursor_field: max( + latest_benchmark, self._field_to_datetime(current_stream_state[self.cursor_field]) + ).to_datetime_string() + } + return {self.cursor_field: latest_benchmark.to_datetime_string()} def request_params(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: @@ -90,12 +108,11 @@ def request_params(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMa ) return params - def path(self, **kwargs) -> str: - return "/export/data.json" - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: for obj in response.iter_lines(): - yield json.loads(obj) + record = json.loads(obj) + record[self.cursor_field] = self._field_to_datetime(record[self.cursor_field]) + yield record class Lists(IterableStream): @@ -106,17 +123,18 @@ def path(self, **kwargs) -> str: class ListUsers(IterableStream): + primary_key = "listId" data_field = "getUsers" name = "list_users" + def path(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> str: + return f"lists/{self.data_field}?listId={stream_slice['list_id']}" + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: lists = Lists(api_key=self._api_key) for list_record in lists.read_records(sync_mode=kwargs.get("sync_mode", SyncMode.full_refresh)): yield {"list_id": list_record["id"]} - def path(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> str: - return f"lists/{self.data_field}?listId={stream_slice['list_id']}" - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: list_id = self._get_list_id(response.url) for user in response.iter_lines(): @@ -138,6 +156,83 @@ def path(self, **kwargs) -> str: return "campaigns" +class CampaignsMetrics(IterableStream): + primary_key = None + data_field = None + + def __init__(self, api_key: str, start_date: str): + """ + https://api.iterable.com/api/docs#campaigns_metrics + """ + super().__init__(api_key) + self.start_date = start_date + + def path(self, **kwargs) -> str: + return "campaigns/metrics" + + def request_params(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> MutableMapping[str, Any]: + params = super().request_params(**kwargs) + params["campaignId"] = stream_slice.get("campaign_ids") + params["startDateTime"] = self.start_date + + return params + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + lists = Campaigns(api_key=self._api_key) + campaign_ids = [] + for list_record in lists.read_records(sync_mode=kwargs.get("sync_mode", SyncMode.full_refresh)): + campaign_ids.append(list_record["id"]) + + if len(campaign_ids) == CAMPAIGNS_PER_REQUEST: + yield {"campaign_ids": campaign_ids} + campaign_ids = [] + + if campaign_ids: + yield {"campaign_ids": campaign_ids} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + content = response.content.decode() + records = self._parse_csv_string_to_dict(content) + + for record in records: + yield {"data": record} + + @staticmethod + def _parse_csv_string_to_dict(csv_string: str) -> List[Dict[str, Any]]: + """ + Parse a response with a csv type to dict object + Example: + csv_string = "a,b,c,d + 1,2,,3 + 6,,1,2" + + output = [{"a": 1, "b": 2, "d": 3}, + {"a": 6, "c": 1, "d": 2}] + + + :param csv_string: API endpoint response with csv format + :return: parsed API response + + """ + + reader = csv.DictReader(StringIO(csv_string), delimiter=",") + result = [] + + for row in reader: + for key, value in row.items(): + if value == "": + continue + try: + row[key] = int(value) + except ValueError: + row[key] = float(value) + row = {k: v for k, v in row.items() if v != ""} + + result.append(row) + + return result + + class Channels(IterableStream): data_field = "channels" @@ -185,6 +280,36 @@ class EmailUnsubscribe(IterableExportStream): data_field = "emailUnsubscribe" +class Events(IterableStream): + """ + https://api.iterable.com/api/docs#events_User_events + """ + primary_key = None + data_field = "events" + page_size = EVENT_ROWS_LIMIT + + def path(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> str: + return f"events/{stream_slice['email']}" + + def request_params(self, **kwargs) -> MutableMapping[str, Any]: + params = super().request_params(**kwargs) + params["limit"] = self.page_size + + return params + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + lists = ListUsers(api_key=self._api_key) + stream_slices = lists.stream_slices() + + for stream_slice in stream_slices: + for list_record in lists.read_records(sync_mode=kwargs.get("sync_mode", SyncMode.full_refresh), stream_slice=stream_slice): + yield {"email": list_record["email"]} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + for record in super().parse_response(response, **kwargs): + yield {"data": record} + + class MessageTypes(IterableStream): data_field = "messageTypes" name = "message_types" @@ -194,6 +319,7 @@ def path(self, **kwargs) -> str: class Metadata(IterableStream): + primary_key = None data_field = "results" def path(self, **kwargs) -> str: @@ -216,7 +342,10 @@ def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwarg def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: response_json = response.json() - yield from response_json.get(self.data_field, []) + records = response_json.get(self.data_field, []) + + for record in records: + yield record class Users(IterableExportStream): diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/campaigns_metrics.json b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/campaigns_metrics.json new file mode 100644 index 000000000000..c7cc50b24427 --- /dev/null +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/campaigns_metrics.json @@ -0,0 +1,8 @@ +{ + "properties": { + "data": { + "type": ["null", "object"] + } + }, + "type": ["null", "object"] +} diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/events.json b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/events.json new file mode 100644 index 000000000000..c7cc50b24427 --- /dev/null +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/events.json @@ -0,0 +1,8 @@ +{ + "properties": { + "data": { + "type": ["null", "object"] + } + }, + "type": ["null", "object"] +} diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/metadata.json b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/metadata.json index 6af7b89d55be..98f35a056efb 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/metadata.json +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/metadata.json @@ -1,67 +1,7 @@ { "properties": { - "table": { + "name": { "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "size": { - "type": ["null", "integer"] - }, - "lastModified": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "inventory": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "sku": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "price": { - "type": ["null", "integer"] - }, - "product_type": { - "type": ["null", "string"] - }, - "compare_at_price": { - "type": ["null", "number"] - }, - "id": { - "type": ["null", "string"] - }, - "product_id": { - "type": ["null", "string"] - }, - "categories": { - "type": ["null", "array"], - "items": {} - }, - "vendor": { - "type": ["null", "string"] - }, - "discount": { - "type": ["null", "integer"] - }, - "imageUrl": { - "type": ["null", "string"] - }, - "handle": { - "type": ["null", "string"] - } - } } }, "type": ["null", "object"] diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/source.py b/airbyte-integrations/connectors/source-iterable/source_iterable/source.py index 3cac52948eee..82aed9ed90c0 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/source.py +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/source.py @@ -11,6 +11,7 @@ from .api import ( Campaigns, + CampaignsMetrics, Channels, EmailBounce, EmailClick, @@ -20,6 +21,7 @@ EmailSendSkip, EmailSubscribe, EmailUnsubscribe, + Events, Lists, ListUsers, MessageTypes, @@ -41,6 +43,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: return [ Campaigns(api_key=config["api_key"]), + CampaignsMetrics(api_key=config["api_key"], start_date=config["start_date"]), Channels(api_key=config["api_key"]), EmailBounce(api_key=config["api_key"], start_date=config["start_date"]), EmailClick(api_key=config["api_key"], start_date=config["start_date"]), @@ -50,6 +53,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: EmailSendSkip(api_key=config["api_key"], start_date=config["start_date"]), EmailSubscribe(api_key=config["api_key"], start_date=config["start_date"]), EmailUnsubscribe(api_key=config["api_key"], start_date=config["start_date"]), + Events(api_key=config["api_key"]), Lists(api_key=config["api_key"]), ListUsers(api_key=config["api_key"]), MessageTypes(api_key=config["api_key"]), diff --git a/docs/integrations/sources/iterable.md b/docs/integrations/sources/iterable.md index ffbea07554ed..9c249c82ef21 100644 --- a/docs/integrations/sources/iterable.md +++ b/docs/integrations/sources/iterable.md @@ -11,6 +11,7 @@ This source can sync data for the [Iterable API](https://api.iterable.com/api/do Several output streams are available from this source: * [Campaigns](https://api.iterable.com/api/docs#campaigns_campaigns) +* [Campaign Metrics](https://api.iterable.com/api/docs#campaigns_metrics) * [Channels](https://api.iterable.com/api/docs#channels_channels) * [Email Bounce](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental sync\) * [Email Click](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental sync\) @@ -20,6 +21,7 @@ Several output streams are available from this source: * [Email Send Skip](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental sync\) * [Email Subscribe](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental sync\) * [Email Unsubscribe](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental sync\) +* [Events](https://api.iterable.com/api/docs#events_User_events) * [Lists](https://api.iterable.com/api/docs#lists_getLists) * [List Users](https://api.iterable.com/api/docs#lists_getLists_0) * [Message Types](https://api.iterable.com/api/docs#messageTypes_messageTypes) @@ -56,4 +58,5 @@ Please read [How to find your API key](https://support.iterable.com/hc/en-us/art | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | -| `0.1.7` | 2021-09-20 | [](https://github.com/airbytehq/airbyte/pull/) | Updated schema for: campaigns, lists, templates, metadata | +| `0.1.8` | 2021-09-20 | [5915](https://github.com/airbytehq/airbyte/pull/5915) | Add new streams: campaign_metrics, events | +| `0.1.7` | 2021-09-20 | [6242](https://github.com/airbytehq/airbyte/pull/6242) | Updated schema for: campaigns, lists, templates, metadata |