From 30f2621e710d7f12a0adbd2fc12f26a8236a0a30 Mon Sep 17 00:00:00 2001 From: Igli Date: Thu, 14 Oct 2021 23:05:24 +0200 Subject: [PATCH 1/5] add source lemlist --- .../789f8e7a-2d28-11ec-8d3d-0242ac130003.json | 7 + .../resources/seed/source_definitions.yaml | 6 + airbyte-integrations/builds.md | 1 + .../connectors/source-lemlist/.dockerignore | 7 + .../connectors/source-lemlist/Dockerfile | 38 +++++ .../connectors/source-lemlist/README.md | 132 ++++++++++++++++++ .../source-lemlist/acceptance-test-config.yml | 23 +++ .../source-lemlist/acceptance-test-docker.sh | 16 +++ .../connectors/source-lemlist/bootstrap.md | 26 ++++ .../connectors/source-lemlist/build.gradle | 14 ++ .../integration_tests/__init__.py | 3 + .../integration_tests/acceptance.py | 14 ++ .../integration_tests/configured_catalog.json | 40 ++++++ .../integration_tests/expected_records.txt | 6 + .../integration_tests/invalid_config.json | 3 + .../integration_tests/sample_config.json | 3 + .../connectors/source-lemlist/main.py | 13 ++ .../source-lemlist/requirements.txt | 2 + .../sample_files/configured_catalog.json | 40 ++++++ .../sample_files/invalid_config.json | 3 + .../sample_files/sample_config.json | 3 + .../connectors/source-lemlist/setup.py | 30 ++++ .../source-lemlist/source_lemlist/__init__.py | 8 ++ .../source-lemlist/source_lemlist/auth.py | 15 ++ .../source_lemlist/schemas/activities.json | 98 +++++++++++++ .../source_lemlist/schemas/campaigns.json | 18 +++ .../source_lemlist/schemas/team.json | 41 ++++++ .../source_lemlist/schemas/unsubscribes.json | 12 ++ .../source-lemlist/source_lemlist/source.py | 44 ++++++ .../source-lemlist/source_lemlist/spec.json | 17 +++ .../source-lemlist/source_lemlist/streams.py | 80 +++++++++++ .../source-lemlist/unit_tests/__init__.py | 3 + .../source-lemlist/unit_tests/test_source.py | 28 ++++ .../source-lemlist/unit_tests/test_streams.py | 105 ++++++++++++++ docs/SUMMARY.md | 1 + docs/integrations/README.md | 1 + docs/integrations/sources/lemlist.md | 39 ++++++ 37 files changed, 940 insertions(+) create mode 100644 airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/789f8e7a-2d28-11ec-8d3d-0242ac130003.json create mode 100644 airbyte-integrations/connectors/source-lemlist/.dockerignore create mode 100644 airbyte-integrations/connectors/source-lemlist/Dockerfile create mode 100644 airbyte-integrations/connectors/source-lemlist/README.md create mode 100644 airbyte-integrations/connectors/source-lemlist/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-lemlist/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-lemlist/bootstrap.md create mode 100644 airbyte-integrations/connectors/source-lemlist/build.gradle create mode 100644 airbyte-integrations/connectors/source-lemlist/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-lemlist/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-lemlist/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-lemlist/integration_tests/expected_records.txt create mode 100644 airbyte-integrations/connectors/source-lemlist/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-lemlist/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-lemlist/main.py create mode 100644 airbyte-integrations/connectors/source-lemlist/requirements.txt create mode 100644 airbyte-integrations/connectors/source-lemlist/sample_files/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-lemlist/sample_files/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-lemlist/sample_files/sample_config.json create mode 100644 airbyte-integrations/connectors/source-lemlist/setup.py create mode 100644 airbyte-integrations/connectors/source-lemlist/source_lemlist/__init__.py create mode 100644 airbyte-integrations/connectors/source-lemlist/source_lemlist/auth.py create mode 100644 airbyte-integrations/connectors/source-lemlist/source_lemlist/schemas/activities.json create mode 100644 airbyte-integrations/connectors/source-lemlist/source_lemlist/schemas/campaigns.json create mode 100644 airbyte-integrations/connectors/source-lemlist/source_lemlist/schemas/team.json create mode 100644 airbyte-integrations/connectors/source-lemlist/source_lemlist/schemas/unsubscribes.json create mode 100644 airbyte-integrations/connectors/source-lemlist/source_lemlist/source.py create mode 100644 airbyte-integrations/connectors/source-lemlist/source_lemlist/spec.json create mode 100644 airbyte-integrations/connectors/source-lemlist/source_lemlist/streams.py create mode 100644 airbyte-integrations/connectors/source-lemlist/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-lemlist/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-lemlist/unit_tests/test_streams.py create mode 100644 docs/integrations/sources/lemlist.md diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/789f8e7a-2d28-11ec-8d3d-0242ac130003.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/789f8e7a-2d28-11ec-8d3d-0242ac130003.json new file mode 100644 index 000000000000..a2dc9b1479a0 --- /dev/null +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/789f8e7a-2d28-11ec-8d3d-0242ac130003.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "789f8e7a-2d28-11ec-8d3d-0242ac130003", + "name": "Lemlist", + "dockerRepository": "airbyte/source-lemlist", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://docs.airbyte.io/integrations/sources/lemlist" +} diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 96565960fe58..fe73bf72c07e 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -171,6 +171,12 @@ dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/klaviyo sourceType: api +- sourceDefinitionId: 789f8e7a-2d28-11ec-8d3d-0242ac130003 + name: Lemlist + dockerRepository: airbyte/source-lemlist + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/source-lemlist + sourceType: api - sourceDefinitionId: 9da77001-af33-4bcd-be46-6252bf9342b9 name: Shopify dockerRepository: airbyte/source-shopify diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index f3d934d438ac..ac2c29108ca8 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -46,6 +46,7 @@ | Kafka | [![source-kafka](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-kafka%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-kafka) | | Klaviyo | [![source-klaviyo](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-klaviyo%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-klaviyo) | | Kustomer | [![source-kustomer-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-kustomer-singer%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-kustomer-singer) | +| Lemlist | [![source-lemlist](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-lemlist%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-lemlist) | | Mailchimp | [![source-mailchimp](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-mailchimp%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-mailchimp) | | Marketo | [![source-marketo](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-marketo%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-marketo) | | Microsoft SQL Server \(MSSQL\) | [![source-mssql](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-mssql%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-mssql) | diff --git a/airbyte-integrations/connectors/source-lemlist/.dockerignore b/airbyte-integrations/connectors/source-lemlist/.dockerignore new file mode 100644 index 000000000000..880323ffc755 --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/.dockerignore @@ -0,0 +1,7 @@ +* +!Dockerfile +!Dockerfile.test +!main.py +!source_lemlist +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-lemlist/Dockerfile b/airbyte-integrations/connectors/source-lemlist/Dockerfile new file mode 100644 index 000000000000..650f683c6886 --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.7.11-alpine3.14 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_lemlist ./source_lemlist + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-lemlist diff --git a/airbyte-integrations/connectors/source-lemlist/README.md b/airbyte-integrations/connectors/source-lemlist/README.md new file mode 100644 index 000000000000..702b50b8c1ea --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/README.md @@ -0,0 +1,132 @@ +# Lemlist Source + +This is the repository for the Lemlist source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/lemlist). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-lemlist:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/lemlist) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_lemlist/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source lemlist test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-lemlist:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-lemlist:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-lemlist:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-lemlist:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-lemlist:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-lemlist:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-lemlist:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-lemlist:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-lemlist/acceptance-test-config.yml b/airbyte-integrations/connectors/source-lemlist/acceptance-test-config.yml new file mode 100644 index 000000000000..caf1871d6d2e --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/acceptance-test-config.yml @@ -0,0 +1,23 @@ +connector_image: airbyte/source-lemlist:dev +tests: + spec: + - spec_path: "source_lemlist/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: ["campaigns", "activities", "unsubscribes"] + expect_records: + path: "integration_tests/expected_records.txt" + extra_fields: no + exact_order: no + extra_records: yes + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-lemlist/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-lemlist/acceptance-test-docker.sh new file mode 100644 index 000000000000..e4d8b1cef896 --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-lemlist/bootstrap.md b/airbyte-integrations/connectors/source-lemlist/bootstrap.md new file mode 100644 index 000000000000..fde382598176 --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/bootstrap.md @@ -0,0 +1,26 @@ +# Lemlist + +## API Reference + +API Docs: https://developer.lemlist.com/#introduction + +## Overview + +Lemlist is your sales automation and cold email software. Using its API you can retrieve information about campaigns, activities and unsubscribes. + +- Lemlist API uses Basic Authentication. +- Pagination is offset-based. +- It uses fixed-window rate limiting strategy. + +## Endpoints + +Lemlist API consists of four endpoints which can be extracted data from: + + 1. **Team**: This endpoint retrieves information of your team. + 2. **Campaigns**: This endpoint retrieves the list of all campaigns. + 3. **Activities**: This endpoint retrieves the last 100 activities. + 4. **Unsubscribes**: This endpoint retrieves the list of all people who are unsubscribed. + +## Notes + +- The API doesn't have any way to filter information so it doesn't support incremental syncs. diff --git a/airbyte-integrations/connectors/source-lemlist/build.gradle b/airbyte-integrations/connectors/source-lemlist/build.gradle new file mode 100644 index 000000000000..f22f0eb5697c --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/build.gradle @@ -0,0 +1,14 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_lemlist' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) + implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-lemlist/integration_tests/__init__.py b/airbyte-integrations/connectors/source-lemlist/integration_tests/__init__.py new file mode 100644 index 000000000000..46b7376756ec --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-lemlist/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-lemlist/integration_tests/acceptance.py new file mode 100644 index 000000000000..108075487440 --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """ This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-lemlist/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-lemlist/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..6fdac40ccc0f --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/integration_tests/configured_catalog.json @@ -0,0 +1,40 @@ +{ + "streams": [ + { + "stream": { + "name": "team", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "campaigns", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "activities", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "unsubscribes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-lemlist/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-lemlist/integration_tests/expected_records.txt new file mode 100644 index 000000000000..bdd9c7165fd5 --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/integration_tests/expected_records.txt @@ -0,0 +1,6 @@ +{"stream": "team", "data": {"_id": "tea_yw9qHaiihCpa8fx5t", "name": "Saky yt's Team", "userIds": ["usr_CTxZq8xXJBBrdgXRm"], "createdBy": "usr_CTxZq8xXJBBrdgXRm", "createdAt": "2021-10-10T21:17:05.174Z", "apiKey": "0aa77935198bb521bf17858c34167d76", "beta": [], "billing": {"freetrialExpiresAt": "2021-10-24T21:17:05.535Z", "ok": true, "plan": "freetrial", "quantity": 1}, "campaignCron": {"lastAt": "2021-10-10T21:17:13.522Z"}, "ctdCheck": {"lastAt": "2021-10-10T21:17:15.450Z"}}, "emitted_at": 1634230668000} +{"stream": "activities", "data": {"_id": "act_2s8H3reh97MH2vqjG", "type": "emailsInterested", "teamId": "tea_yw9qHaiihCpa8fx5t", "createdAt": "2021-10-10T21:23:08.482Z", "leadId": "lea_L5RZqF7bu7aHCPe5s", "leadEmail": "nadja+ob@lemlist.com", "campaignId": "cam_osLYq2K9dZ8Ya4Zr2", "createdBy": "usr_CTxZq8xXJBBrdgXRm", "leadFirstName": "Nadja", "leadLastName": "Komnenic", "sequenceId": "seq_je4nnaXGQ4dDy6uvS", "isFirst": true, "sequenceStep": 0, "emailTemplateId": "etp_nA35CEhmunFcPp9Q5", "extra": {"userName": "Saky yt", "leadPhone": "+38112345678", "sendUserId": "usr_CTxZq8xXJBBrdgXRm", "leadPicture": "https://app.lemlist.com/onboarding/nadja.jpg", "leadCompanyName": "lemlist"}, "firstName": "Nadja", "lastName": "Komnenic", "email": "nadja+ob@lemlist.com", "companyName": "lemlist", "picture": "https://app.lemlist.com/onboarding/nadja.jpg", "phone": "+38112345678", "linkedinUrl": "https://www.linkedin.com/in/nadjakomnenic/", "icebreaker": "Honestly I'm a huge fan of your content, especially when it comes to your growth strategy as a bootstrapped startup.", "campaignName": "Saky's 1st campaign", "emailTemplateName": "onboardingEmail"}, "emitted_at": 1634230669000} +{"stream": "activities", "data": {"_id": "act_47zhZ2SJAaPPtMiSW", "type": "emailsReplied", "teamId": "tea_yw9qHaiihCpa8fx5t", "createdAt": "2021-10-10T21:22:41.246Z", "leadId": "lea_L5RZqF7bu7aHCPe5s", "leadEmail": "nadja+ob@lemlist.com", "campaignId": "cam_osLYq2K9dZ8Ya4Zr2", "createdBy": "usr_CTxZq8xXJBBrdgXRm", "leadFirstName": "Nadja", "leadLastName": "Komnenic", "sequenceId": "seq_je4nnaXGQ4dDy6uvS", "isFirst": true, "sequenceStep": 0, "emailTemplateId": "etp_nA35CEhmunFcPp9Q5", "extra": {"emailId": "eml_vhXQnHXHnJaEic29G", "stopped": true, "leadPhone": "+38112345678", "messageId": "", "sendUserId": "usr_CTxZq8xXJBBrdgXRm", "leadPicture": "https://app.lemlist.com/onboarding/nadja.jpg", "sendUserName": "Saky YT", "sendUserEmail": "saky_yt@outlook.com", "leadCompanyName": "lemlist"}, "firstName": "Nadja", "lastName": "Komnenic", "email": "nadja+ob@lemlist.com", "companyName": "lemlist", "picture": "https://app.lemlist.com/onboarding/nadja.jpg", "phone": "+38112345678", "linkedinUrl": "https://www.linkedin.com/in/nadjakomnenic/", "icebreaker": "Honestly I'm a huge fan of your content, especially when it comes to your growth strategy as a bootstrapped startup.", "campaignName": "Saky's 1st campaign", "emailTemplateName": "onboardingEmail"}, "emitted_at": 1634230669000} +{"stream": "activities", "data": {"_id": "act_CDQdevhdThaxfu6eh", "type": "emailsOpened", "teamId": "tea_yw9qHaiihCpa8fx5t", "createdAt": "2021-10-10T21:22:40.242Z", "leadId": "lea_L5RZqF7bu7aHCPe5s", "leadEmail": "nadja+ob@lemlist.com", "campaignId": "cam_osLYq2K9dZ8Ya4Zr2", "createdBy": "usr_CTxZq8xXJBBrdgXRm", "leadFirstName": "Nadja", "leadLastName": "Komnenic", "sequenceId": "seq_je4nnaXGQ4dDy6uvS", "isFirst": false, "sequenceStep": 0, "emailTemplateId": "etp_nA35CEhmunFcPp9Q5", "extra": {"stopped": true, "leadPhone": "+38112345678", "sendUserId": "usr_CTxZq8xXJBBrdgXRm", "leadPicture": "https://app.lemlist.com/onboarding/nadja.jpg", "sendUserName": "Saky YT", "sendUserEmail": "saky_yt@outlook.com", "leadCompanyName": "lemlist"}, "firstName": "Nadja", "lastName": "Komnenic", "email": "nadja+ob@lemlist.com", "companyName": "lemlist", "picture": "https://app.lemlist.com/onboarding/nadja.jpg", "phone": "+38112345678", "linkedinUrl": "https://www.linkedin.com/in/nadjakomnenic/", "icebreaker": "Honestly I'm a huge fan of your content, especially when it comes to your growth strategy as a bootstrapped startup.", "campaignName": "Saky's 1st campaign", "emailTemplateName": "onboardingEmail"}, "emitted_at": 1634230669000} +{"stream": "activities", "data": {"_id": "act_vMSDcBFqNxsgSZxvu", "type": "emailsOpened", "teamId": "tea_yw9qHaiihCpa8fx5t", "createdAt": "2021-10-10T21:22:37.859Z", "leadId": "lea_L5RZqF7bu7aHCPe5s", "leadEmail": "nadja+ob@lemlist.com", "campaignId": "cam_osLYq2K9dZ8Ya4Zr2", "createdBy": "usr_CTxZq8xXJBBrdgXRm", "leadFirstName": "Nadja", "leadLastName": "Komnenic", "sequenceId": "seq_je4nnaXGQ4dDy6uvS", "isFirst": true, "sequenceStep": 0, "emailTemplateId": "etp_nA35CEhmunFcPp9Q5", "extra": {"leadPhone": "+38112345678", "sendUserId": "usr_CTxZq8xXJBBrdgXRm", "leadPicture": "https://app.lemlist.com/onboarding/nadja.jpg", "sendUserName": "Saky YT", "sendUserEmail": "saky_yt@outlook.com", "leadCompanyName": "lemlist"}, "firstName": "Nadja", "lastName": "Komnenic", "email": "nadja+ob@lemlist.com", "companyName": "lemlist", "picture": "https://app.lemlist.com/onboarding/nadja.jpg", "phone": "+38112345678", "linkedinUrl": "https://www.linkedin.com/in/nadjakomnenic/", "icebreaker": "Honestly I'm a huge fan of your content, especially when it comes to your growth strategy as a bootstrapped startup.", "campaignName": "Saky's 1st campaign", "emailTemplateName": "onboardingEmail"}, "emitted_at": 1634230669000} +{"stream": "activities", "data": {"_id": "act_kgSBWPrNPaHS7qE7S", "type": "emailsSent", "teamId": "tea_yw9qHaiihCpa8fx5t", "createdAt": "2021-10-10T21:22:34.427Z", "leadId": "lea_L5RZqF7bu7aHCPe5s", "leadEmail": "nadja+ob@lemlist.com", "campaignId": "cam_osLYq2K9dZ8Ya4Zr2", "createdBy": "usr_CTxZq8xXJBBrdgXRm", "leadFirstName": "Nadja", "leadLastName": "Komnenic", "sequenceId": "seq_je4nnaXGQ4dDy6uvS", "isFirst": true, "sequenceStep": 0, "emailTemplateId": "etp_nA35CEhmunFcPp9Q5", "extra": {"emailId": "eml_dkEWJr8MFGMXPbgy8", "leadPhone": "+38112345678", "messageId": "", "sendUserId": "usr_CTxZq8xXJBBrdgXRm", "leadPicture": "https://app.lemlist.com/onboarding/nadja.jpg", "sendUserName": "Saky YT", "sendUserEmail": "saky_yt@outlook.com", "leadCompanyName": "lemlist"}, "firstName": "Nadja", "lastName": "Komnenic", "email": "nadja+ob@lemlist.com", "companyName": "lemlist", "picture": "https://app.lemlist.com/onboarding/nadja.jpg", "phone": "+38112345678", "linkedinUrl": "https://www.linkedin.com/in/nadjakomnenic/", "icebreaker": "Honestly I'm a huge fan of your content, especially when it comes to your growth strategy as a bootstrapped startup.", "campaignName": "Saky's 1st campaign", "emailTemplateName": "onboardingEmail"}, "emitted_at": 1634230669000} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-lemlist/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-lemlist/integration_tests/invalid_config.json new file mode 100644 index 000000000000..89993539f0df --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "api_key": "" +} diff --git a/airbyte-integrations/connectors/source-lemlist/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-lemlist/integration_tests/sample_config.json new file mode 100644 index 000000000000..73112f5a25a7 --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "api_key": "" +} diff --git a/airbyte-integrations/connectors/source-lemlist/main.py b/airbyte-integrations/connectors/source-lemlist/main.py new file mode 100644 index 000000000000..2951ed0cb4c5 --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_lemlist import SourceLemlist + +if __name__ == "__main__": + source = SourceLemlist() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-lemlist/requirements.txt b/airbyte-integrations/connectors/source-lemlist/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-lemlist/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-lemlist/sample_files/configured_catalog.json new file mode 100644 index 000000000000..6fdac40ccc0f --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/sample_files/configured_catalog.json @@ -0,0 +1,40 @@ +{ + "streams": [ + { + "stream": { + "name": "team", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "campaigns", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "activities", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "unsubscribes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-lemlist/sample_files/invalid_config.json b/airbyte-integrations/connectors/source-lemlist/sample_files/invalid_config.json new file mode 100644 index 000000000000..89993539f0df --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/sample_files/invalid_config.json @@ -0,0 +1,3 @@ +{ + "api_key": "" +} diff --git a/airbyte-integrations/connectors/source-lemlist/sample_files/sample_config.json b/airbyte-integrations/connectors/source-lemlist/sample_files/sample_config.json new file mode 100644 index 000000000000..73112f5a25a7 --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/sample_files/sample_config.json @@ -0,0 +1,3 @@ +{ + "api_key": "" +} diff --git a/airbyte-integrations/connectors/source-lemlist/setup.py b/airbyte-integrations/connectors/source-lemlist/setup.py new file mode 100644 index 000000000000..5d334dc1a5ea --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/setup.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.2.5", + "pytest-mock~=3.6.1", + "source-acceptance-test", + "responses~=0.14.0" +] + +setup( + name="source_lemlist", + description="Source implementation for Lemlist.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/__init__.py b/airbyte-integrations/connectors/source-lemlist/source_lemlist/__init__.py new file mode 100644 index 000000000000..e7567c6c38da --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/source_lemlist/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceLemlist + +__all__ = ["SourceLemlist"] diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/auth.py b/airbyte-integrations/connectors/source-lemlist/source_lemlist/auth.py new file mode 100644 index 000000000000..f7f3ccd81b07 --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/source_lemlist/auth.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import base64 +from typing import Tuple + +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + + +class HttpBasicAuthenticator(TokenAuthenticator): + def __init__(self, auth: Tuple[str, str], auth_method: str = "Basic", **kwargs): + auth_string = f"{auth[0]}:{auth[1]}".encode("utf8") + b64_encoded = base64.b64encode(auth_string).decode("utf8") + super().__init__(token=b64_encoded, auth_method=auth_method, **kwargs) diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/schemas/activities.json b/airbyte-integrations/connectors/source-lemlist/source_lemlist/schemas/activities.json new file mode 100644 index 000000000000..4b3e8d7e0435 --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/source_lemlist/schemas/activities.json @@ -0,0 +1,98 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "_id": { + "type": "string" + }, + "type": { + "type": ["null", "string"] + }, + "teamId": { + "type": ["null", "string"] + }, + "createdAt": { + "type": ["null", "string"] + }, + "leadId": { + "type": ["null", "string"] + }, + "leadEmail": { + "type": ["null", "string"] + }, + "campaignId": { + "type": ["null", "string"] + }, + "createdBy": { + "type": ["null", "string"] + }, + "leadFirstName": { + "type": ["null", "string"] + }, + "leadLastName": { + "type": ["null", "string"] + }, + "sequenceId": { + "type": ["null", "string"] + }, + "isFirst": { + "type": ["null", "boolean"] + }, + "sequenceStep": { + "type": ["null", "integer"] + }, + "emailTemplateId": { + "type": ["null", "string"] + }, + "extra": { + "type": ["null", "object"], + "properties": { + "userName": { + "type": ["null", "string"] + }, + "leadPhone": { + "type": ["null", "string"] + }, + "sendUserId": { + "type": ["null", "string"] + }, + "leadPicture": { + "type": ["null", "string"] + }, + "leadCompanyName": { + "type": ["null", "string"] + } + } + }, + "firstName": { + "type": ["null", "string"] + }, + "lastName": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "companyName": { + "type": ["null", "string"] + }, + "picture": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "linkedinUrl": { + "type": ["null", "string"] + }, + "icebreaker": { + "type": ["null", "string"] + }, + "campaignName": { + "type": ["null", "string"] + }, + "emailTemplateName": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/schemas/campaigns.json b/airbyte-integrations/connectors/source-lemlist/source_lemlist/schemas/campaigns.json new file mode 100644 index 000000000000..5a29bc49c33b --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/source_lemlist/schemas/campaigns.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "_id": { + "type": "string" + }, + "name": { + "type": ["null", "string"] + }, + "labels": { + "type": ["null", "array"], + "items": { + "type": "string" + } + } + } +} diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/schemas/team.json b/airbyte-integrations/connectors/source-lemlist/source_lemlist/schemas/team.json new file mode 100644 index 000000000000..5fe12ac770c5 --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/source_lemlist/schemas/team.json @@ -0,0 +1,41 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "_id": { + "type": "string" + }, + "name": { + "type": ["null", "string"] + }, + "userIds": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "createdBy": { + "type": ["null", "string"] + }, + "createdAt": { + "type": ["null", "string"] + }, + "apiKey": { + "type": ["null", "string"] + }, + "billing": { + "type": ["null", "object"], + "properties": { + "quantity": { + "type": ["null", "integer"] + }, + "ok": { + "type": ["null", "boolean"] + }, + "plan": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/schemas/unsubscribes.json b/airbyte-integrations/connectors/source-lemlist/source_lemlist/schemas/unsubscribes.json new file mode 100644 index 000000000000..19e8ffa1ac12 --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/source_lemlist/schemas/unsubscribes.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "_id": { + "type": "string" + }, + "email": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/source.py b/airbyte-integrations/connectors/source-lemlist/source_lemlist/source.py new file mode 100644 index 000000000000..1230955dacc2 --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/source_lemlist/source.py @@ -0,0 +1,44 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from typing import Any, List, Mapping, Tuple + +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from source_lemlist.auth import HttpBasicAuthenticator +from airbyte_cdk.models import SyncMode + +from .streams import Team, Campaigns, Activities, Unsubscribes + + +class SourceLemlist(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + try: + auth = HttpBasicAuthenticator( + ( + "", + config["api_key"], + ), + ) + + team_stream = Team(authenticator=auth) + team_gen = team_stream.read_records(sync_mode=SyncMode.full_refresh) + + next(team_gen) + return True, None + except Exception as error: + return False, f"The provided API key {config['api_key']} is invalid. - {repr(error)}" + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + auth = HttpBasicAuthenticator( + ( + "", + config["api_key"], + ), + ) + return [Team(authenticator=auth), + Campaigns(authenticator=auth), + Activities(authenticator=auth), + Unsubscribes(authenticator=auth)] diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/spec.json b/airbyte-integrations/connectors/source-lemlist/source_lemlist/spec.json new file mode 100644 index 000000000000..c0604784a830 --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/source_lemlist/spec.json @@ -0,0 +1,17 @@ +{ + "documentationUrl": "https://docsurl.com", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Lemlist Spec", + "type": "object", + "required": ["api_key"], + "additionalProperties": false, + "properties": { + "api_key": { + "type": "string", + "description": "API key to access your lemlist account.", + "airbyte_secret": true + } + } + } +} diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/streams.py b/airbyte-integrations/connectors/source-lemlist/source_lemlist/streams.py new file mode 100644 index 000000000000..9085377f29e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/source_lemlist/streams.py @@ -0,0 +1,80 @@ +from typing import Any, Iterable, Mapping, MutableMapping, Optional + +import requests +from airbyte_cdk.sources.streams.http import HttpStream + + +LEMLIST_API_URL_BASE = "https://api.lemlist.com/api/" + + +class LemlistStream(HttpStream): + """Default and max value page_size can have is 100""" + + url_base = LEMLIST_API_URL_BASE + primary_key = "_id" + page_size = 100 + initial_offset = 0 + offset = initial_offset + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + if len(response.json()) == self.page_size: + self.offset += self.page_size + next_page_params = {"offset": self.offset} + return next_page_params + return None + + def path(self, **kwargs) -> str: + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None, **kwargs + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) + params["offset"] = self.initial_offset + params["limit"] = self.page_size + if next_page_token: + params.update(**next_page_token) + return params + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + records = response.json() + yield from records + + def backoff_time(self, response: requests.Response): + if "Retry-After" in response.headers: + return int(response.headers["Retry-After"]) + else: + self.logger.info("Retry-after header not found. Using default backoff value") + return 2 + +class Team(LemlistStream): + """https://developer.lemlist.com/#get-team-information""" + + def path(self, **kwargs) -> str: + return "team" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + records = response.json() + yield from [records] + +class Campaigns(LemlistStream): + """https://developer.lemlist.com/#campaigns""" + + def path(self, **kwargs) -> str: + return "campaigns" + +class Activities(LemlistStream): + """https://developer.lemlist.com/#activities""" + + def path(self, **kwargs) -> str: + return "activities" + +class Unsubscribes(LemlistStream): + """https://developer.lemlist.com/#unsubscribes""" + + def path(self, **kwargs) -> str: + return "unsubscribes" + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-lemlist/unit_tests/__init__.py b/airbyte-integrations/connectors/source-lemlist/unit_tests/__init__.py new file mode 100644 index 000000000000..46b7376756ec --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-lemlist/unit_tests/test_source.py b/airbyte-integrations/connectors/source-lemlist/unit_tests/test_source.py new file mode 100644 index 000000000000..a49c3e93763d --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/unit_tests/test_source.py @@ -0,0 +1,28 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock, patch + +from source_lemlist.source import SourceLemlist + + +@patch("source_lemlist.source.Team.read_records", return_value = iter(["item"])) +def test_check_connection(_): + test_config = { + "api_key": "test-api-key" + } + logger_mock = MagicMock() + + source = SourceLemlist() + valid_connection, error = source.check_connection(logger_mock, test_config) + + assert valid_connection + + +def test_streams(): + source = SourceLemlist() + config_mock = MagicMock() + streams = source.streams(config_mock) + expected_streams_number = 4 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-lemlist/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-lemlist/unit_tests/test_streams.py new file mode 100644 index 000000000000..6eb0c55241a4 --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/unit_tests/test_streams.py @@ -0,0 +1,105 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +import requests +import responses +from source_lemlist.streams import LemlistStream + + +def setup_responses(): + responses.add( + responses.GET, + "https://api.lemlist.com/api/example_endpoint", + json=[ + { + "_id": "cam_aaWL92T22Sei3Bz6v", + "name": "Campaign1", + "labels": ["label 1", "label 2"] + }, + { + "_id": "cam_aaXwBiebA8pWPKqpK", + "name": "Campaign2" + } + ], + ) + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(LemlistStream, "path", "v0/example_endpoint") + mocker.patch.object(LemlistStream, "primary_key", "test_primary_key") + mocker.patch.object(LemlistStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = LemlistStream() + inputs = { + "stream_slice": None, + "stream_state": None, + "next_page_token": {"offset": 100}, + } + expected_params = {"limit": stream.page_size, "offset": 100} + assert stream.request_params(**inputs) == expected_params + + +@responses.activate +def test_next_page_token(patch_base_class): + setup_responses() + stream = LemlistStream() + inputs = {"response": requests.get("https://api.lemlist.com/api/example_endpoint")} + expected_token = None + assert stream.next_page_token(**inputs) == expected_token + + +@responses.activate +def test_parse_response(patch_base_class): + setup_responses() + stream = LemlistStream() + inputs = {"response": requests.get("https://api.lemlist.com/api/example_endpoint")} + expected_parsed_object = { + "_id": "cam_aaWL92T22Sei3Bz6v", + "name": "Campaign1", + "labels": ["label 1", "label 2"] + } + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = LemlistStream() + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": {"offset": 100}} + assert stream.request_headers(**inputs) == {} + + +def test_http_method(patch_base_class): + stream = LemlistStream() + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = LemlistStream() + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = LemlistStream() + expected_backoff_time = 2 + assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index b9d998467e2d..261e76b21123 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -74,6 +74,7 @@ * [Kafka](integrations/sources/kafka.md) * [Klaviyo](integrations/sources/klaviyo.md) * [Kustomer](integrations/sources/kustomer.md) + * [Lemlist](integrations/sources/lemlist.md) * [LinkedIn Ads](integrations/sources/linkedin-ads.md) * [Lever Hiring](integrations/sources/lever-hiring.md) * [Looker](integrations/sources/looker.md) diff --git a/docs/integrations/README.md b/docs/integrations/README.md index 77a0b2690e6e..1b85637361cf 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -56,6 +56,7 @@ Airbyte uses a grading system for connectors to help users understand what to ex | [Jira](sources/jira.md) | Certified | | [Klaviyo](sources/klaviyo.md) | Beta | | [Klaviyo](sources/kustomer.md) | Alpha | +| [Lemlist](sources/lemlist.md) | Alpha | | [LinkedIn Ads](sources/linkedin-ads.md) | Beta | | [Kustomer](sources/kustomer.md) | Alpha | | [Lever Hiring](sources/lever-hiring.md) | Beta | diff --git a/docs/integrations/sources/lemlist.md b/docs/integrations/sources/lemlist.md new file mode 100644 index 000000000000..1bce756e6231 --- /dev/null +++ b/docs/integrations/sources/lemlist.md @@ -0,0 +1,39 @@ +# Lemlist + +## Sync overview + +The Lemlist source supports Full Refresh syncs only. + +This source can sync data for the [Lemlist API](https://developer.lemlist.com/#introduction). + +### Output schema + +This Source is capable of syncing the following core Streams: + +* [Team](https://api.lemlist.com/api/team) +* [Campaigns](https://api.lemlist.com/api/campaigns) +* [Activities](chttps://api.lemlist.com/api/activities) +* [Unsubscribes](https://api.lemlist.com/api/unsubscribes) + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :--- | :--- | :--- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | No | | +| Namespaces | No | | + +### Performance considerations + +The Lemlist connector should not run into Lemlist API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. + +## Getting started + +### Requirements + +* Lemlist API key + +## Changelog + +| Version | Date | Pull Request | Subject | +| :--- | :--- | :--- | :--- | From 23552e5a42c6e3ec4a57dfa13747b14cc9878f17 Mon Sep 17 00:00:00 2001 From: Igli Date: Thu, 14 Oct 2021 23:18:02 +0200 Subject: [PATCH 2/5] update change log --- docs/integrations/sources/lemlist.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrations/sources/lemlist.md b/docs/integrations/sources/lemlist.md index 1bce756e6231..1e2c57c3c44b 100644 --- a/docs/integrations/sources/lemlist.md +++ b/docs/integrations/sources/lemlist.md @@ -37,3 +37,4 @@ The Lemlist connector should not run into Lemlist API limitations under normal u | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.0 | 2021-10-14 | [7062](https://github.com/airbytehq/airbyte/pull/7062) | Initial Release | From a0db2326ad8108903c3a1bcba2852b71cd0722a5 Mon Sep 17 00:00:00 2001 From: Igli Date: Thu, 21 Oct 2021 00:21:05 +0200 Subject: [PATCH 3/5] apply default formating --- .../connectors/source-lemlist/setup.py | 7 +------ .../source-lemlist/source_lemlist/source.py | 11 ++++------- .../source-lemlist/source_lemlist/streams.py | 10 ++++++++-- .../source-lemlist/unit_tests/test_source.py | 6 ++---- .../source-lemlist/unit_tests/test_streams.py | 17 +++-------------- 5 files changed, 18 insertions(+), 33 deletions(-) diff --git a/airbyte-integrations/connectors/source-lemlist/setup.py b/airbyte-integrations/connectors/source-lemlist/setup.py index 5d334dc1a5ea..b54f3371b817 100644 --- a/airbyte-integrations/connectors/source-lemlist/setup.py +++ b/airbyte-integrations/connectors/source-lemlist/setup.py @@ -9,12 +9,7 @@ "airbyte-cdk", ] -TEST_REQUIREMENTS = [ - "pytest~=6.2.5", - "pytest-mock~=3.6.1", - "source-acceptance-test", - "responses~=0.14.0" -] +TEST_REQUIREMENTS = ["pytest~=6.2.5", "pytest-mock~=3.6.1", "source-acceptance-test", "responses~=0.14.0"] setup( name="source_lemlist", diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/source.py b/airbyte-integrations/connectors/source-lemlist/source_lemlist/source.py index 1230955dacc2..52b3e8d9b53b 100644 --- a/airbyte-integrations/connectors/source-lemlist/source_lemlist/source.py +++ b/airbyte-integrations/connectors/source-lemlist/source_lemlist/source.py @@ -5,12 +5,12 @@ from typing import Any, List, Mapping, Tuple +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from source_lemlist.auth import HttpBasicAuthenticator -from airbyte_cdk.models import SyncMode -from .streams import Team, Campaigns, Activities, Unsubscribes +from .streams import Activities, Campaigns, Team, Unsubscribes class SourceLemlist(AbstractSource): @@ -25,7 +25,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: team_stream = Team(authenticator=auth) team_gen = team_stream.read_records(sync_mode=SyncMode.full_refresh) - + next(team_gen) return True, None except Exception as error: @@ -38,7 +38,4 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: config["api_key"], ), ) - return [Team(authenticator=auth), - Campaigns(authenticator=auth), - Activities(authenticator=auth), - Unsubscribes(authenticator=auth)] + return [Team(authenticator=auth), Campaigns(authenticator=auth), Activities(authenticator=auth), Unsubscribes(authenticator=auth)] diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/streams.py b/airbyte-integrations/connectors/source-lemlist/source_lemlist/streams.py index 9085377f29e5..f6ba585b8a50 100644 --- a/airbyte-integrations/connectors/source-lemlist/source_lemlist/streams.py +++ b/airbyte-integrations/connectors/source-lemlist/source_lemlist/streams.py @@ -1,9 +1,12 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + from typing import Any, Iterable, Mapping, MutableMapping, Optional import requests from airbyte_cdk.sources.streams.http import HttpStream - LEMLIST_API_URL_BASE = "https://api.lemlist.com/api/" @@ -50,6 +53,7 @@ def backoff_time(self, response: requests.Response): self.logger.info("Retry-after header not found. Using default backoff value") return 2 + class Team(LemlistStream): """https://developer.lemlist.com/#get-team-information""" @@ -60,21 +64,23 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp records = response.json() yield from [records] + class Campaigns(LemlistStream): """https://developer.lemlist.com/#campaigns""" def path(self, **kwargs) -> str: return "campaigns" + class Activities(LemlistStream): """https://developer.lemlist.com/#activities""" def path(self, **kwargs) -> str: return "activities" + class Unsubscribes(LemlistStream): """https://developer.lemlist.com/#unsubscribes""" def path(self, **kwargs) -> str: return "unsubscribes" - \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-lemlist/unit_tests/test_source.py b/airbyte-integrations/connectors/source-lemlist/unit_tests/test_source.py index a49c3e93763d..1309156409ab 100644 --- a/airbyte-integrations/connectors/source-lemlist/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-lemlist/unit_tests/test_source.py @@ -7,11 +7,9 @@ from source_lemlist.source import SourceLemlist -@patch("source_lemlist.source.Team.read_records", return_value = iter(["item"])) +@patch("source_lemlist.source.Team.read_records", return_value=iter(["item"])) def test_check_connection(_): - test_config = { - "api_key": "test-api-key" - } + test_config = {"api_key": "test-api-key"} logger_mock = MagicMock() source = SourceLemlist() diff --git a/airbyte-integrations/connectors/source-lemlist/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-lemlist/unit_tests/test_streams.py index 6eb0c55241a4..b9631f9414d3 100644 --- a/airbyte-integrations/connectors/source-lemlist/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-lemlist/unit_tests/test_streams.py @@ -16,15 +16,8 @@ def setup_responses(): responses.GET, "https://api.lemlist.com/api/example_endpoint", json=[ - { - "_id": "cam_aaWL92T22Sei3Bz6v", - "name": "Campaign1", - "labels": ["label 1", "label 2"] - }, - { - "_id": "cam_aaXwBiebA8pWPKqpK", - "name": "Campaign2" - } + {"_id": "cam_aaWL92T22Sei3Bz6v", "name": "Campaign1", "labels": ["label 1", "label 2"]}, + {"_id": "cam_aaXwBiebA8pWPKqpK", "name": "Campaign2"}, ], ) @@ -62,11 +55,7 @@ def test_parse_response(patch_base_class): setup_responses() stream = LemlistStream() inputs = {"response": requests.get("https://api.lemlist.com/api/example_endpoint")} - expected_parsed_object = { - "_id": "cam_aaWL92T22Sei3Bz6v", - "name": "Campaign1", - "labels": ["label 1", "label 2"] - } + expected_parsed_object = {"_id": "cam_aaWL92T22Sei3Bz6v", "name": "Campaign1", "labels": ["label 1", "label 2"]} assert next(stream.parse_response(**inputs)) == expected_parsed_object From 3b139726db914be5790b05f5a6f85891f6e6f178 Mon Sep 17 00:00:00 2001 From: Igli Date: Fri, 22 Oct 2021 11:16:09 +0200 Subject: [PATCH 4/5] add comment, delete unnecessary files --- .../source-lemlist/sample_files/invalid_config.json | 3 --- .../connectors/source-lemlist/sample_files/sample_config.json | 3 --- .../connectors/source-lemlist/source_lemlist/streams.py | 4 ++++ 3 files changed, 4 insertions(+), 6 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-lemlist/sample_files/invalid_config.json delete mode 100644 airbyte-integrations/connectors/source-lemlist/sample_files/sample_config.json diff --git a/airbyte-integrations/connectors/source-lemlist/sample_files/invalid_config.json b/airbyte-integrations/connectors/source-lemlist/sample_files/invalid_config.json deleted file mode 100644 index 89993539f0df..000000000000 --- a/airbyte-integrations/connectors/source-lemlist/sample_files/invalid_config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "api_key": "" -} diff --git a/airbyte-integrations/connectors/source-lemlist/sample_files/sample_config.json b/airbyte-integrations/connectors/source-lemlist/sample_files/sample_config.json deleted file mode 100644 index 73112f5a25a7..000000000000 --- a/airbyte-integrations/connectors/source-lemlist/sample_files/sample_config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "api_key": "" -} diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/streams.py b/airbyte-integrations/connectors/source-lemlist/source_lemlist/streams.py index f6ba585b8a50..9dccff2449b6 100644 --- a/airbyte-integrations/connectors/source-lemlist/source_lemlist/streams.py +++ b/airbyte-integrations/connectors/source-lemlist/source_lemlist/streams.py @@ -23,6 +23,10 @@ def __init__(self, **kwargs): super().__init__(**kwargs) def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """Pagination is offset-based and response doesn't contain a next_page_token + Thus, the only way to know if there are any more pages is to check if the + number of items in current page is equal to the page_size limit""" + if len(response.json()) == self.page_size: self.offset += self.page_size next_page_params = {"offset": self.offset} From 901726e285c7c39430786f17666518ae69e44e50 Mon Sep 17 00:00:00 2001 From: Igli Date: Tue, 26 Oct 2021 00:44:00 +0200 Subject: [PATCH 5/5] remove unnecessary things --- .../connectors/source-lemlist/acceptance-test-config.yml | 5 ----- .../source-lemlist/integration_tests/expected_records.txt | 6 ------ .../connectors/source-lemlist/source_lemlist/streams.py | 4 +--- 3 files changed, 1 insertion(+), 14 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-lemlist/integration_tests/expected_records.txt diff --git a/airbyte-integrations/connectors/source-lemlist/acceptance-test-config.yml b/airbyte-integrations/connectors/source-lemlist/acceptance-test-config.yml index caf1871d6d2e..39155271a6b6 100644 --- a/airbyte-integrations/connectors/source-lemlist/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-lemlist/acceptance-test-config.yml @@ -13,11 +13,6 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: ["campaigns", "activities", "unsubscribes"] - expect_records: - path: "integration_tests/expected_records.txt" - extra_fields: no - exact_order: no - extra_records: yes full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-lemlist/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-lemlist/integration_tests/expected_records.txt deleted file mode 100644 index bdd9c7165fd5..000000000000 --- a/airbyte-integrations/connectors/source-lemlist/integration_tests/expected_records.txt +++ /dev/null @@ -1,6 +0,0 @@ -{"stream": "team", "data": {"_id": "tea_yw9qHaiihCpa8fx5t", "name": "Saky yt's Team", "userIds": ["usr_CTxZq8xXJBBrdgXRm"], "createdBy": "usr_CTxZq8xXJBBrdgXRm", "createdAt": "2021-10-10T21:17:05.174Z", "apiKey": "0aa77935198bb521bf17858c34167d76", "beta": [], "billing": {"freetrialExpiresAt": "2021-10-24T21:17:05.535Z", "ok": true, "plan": "freetrial", "quantity": 1}, "campaignCron": {"lastAt": "2021-10-10T21:17:13.522Z"}, "ctdCheck": {"lastAt": "2021-10-10T21:17:15.450Z"}}, "emitted_at": 1634230668000} -{"stream": "activities", "data": {"_id": "act_2s8H3reh97MH2vqjG", "type": "emailsInterested", "teamId": "tea_yw9qHaiihCpa8fx5t", "createdAt": "2021-10-10T21:23:08.482Z", "leadId": "lea_L5RZqF7bu7aHCPe5s", "leadEmail": "nadja+ob@lemlist.com", "campaignId": "cam_osLYq2K9dZ8Ya4Zr2", "createdBy": "usr_CTxZq8xXJBBrdgXRm", "leadFirstName": "Nadja", "leadLastName": "Komnenic", "sequenceId": "seq_je4nnaXGQ4dDy6uvS", "isFirst": true, "sequenceStep": 0, "emailTemplateId": "etp_nA35CEhmunFcPp9Q5", "extra": {"userName": "Saky yt", "leadPhone": "+38112345678", "sendUserId": "usr_CTxZq8xXJBBrdgXRm", "leadPicture": "https://app.lemlist.com/onboarding/nadja.jpg", "leadCompanyName": "lemlist"}, "firstName": "Nadja", "lastName": "Komnenic", "email": "nadja+ob@lemlist.com", "companyName": "lemlist", "picture": "https://app.lemlist.com/onboarding/nadja.jpg", "phone": "+38112345678", "linkedinUrl": "https://www.linkedin.com/in/nadjakomnenic/", "icebreaker": "Honestly I'm a huge fan of your content, especially when it comes to your growth strategy as a bootstrapped startup.", "campaignName": "Saky's 1st campaign", "emailTemplateName": "onboardingEmail"}, "emitted_at": 1634230669000} -{"stream": "activities", "data": {"_id": "act_47zhZ2SJAaPPtMiSW", "type": "emailsReplied", "teamId": "tea_yw9qHaiihCpa8fx5t", "createdAt": "2021-10-10T21:22:41.246Z", "leadId": "lea_L5RZqF7bu7aHCPe5s", "leadEmail": "nadja+ob@lemlist.com", "campaignId": "cam_osLYq2K9dZ8Ya4Zr2", "createdBy": "usr_CTxZq8xXJBBrdgXRm", "leadFirstName": "Nadja", "leadLastName": "Komnenic", "sequenceId": "seq_je4nnaXGQ4dDy6uvS", "isFirst": true, "sequenceStep": 0, "emailTemplateId": "etp_nA35CEhmunFcPp9Q5", "extra": {"emailId": "eml_vhXQnHXHnJaEic29G", "stopped": true, "leadPhone": "+38112345678", "messageId": "", "sendUserId": "usr_CTxZq8xXJBBrdgXRm", "leadPicture": "https://app.lemlist.com/onboarding/nadja.jpg", "sendUserName": "Saky YT", "sendUserEmail": "saky_yt@outlook.com", "leadCompanyName": "lemlist"}, "firstName": "Nadja", "lastName": "Komnenic", "email": "nadja+ob@lemlist.com", "companyName": "lemlist", "picture": "https://app.lemlist.com/onboarding/nadja.jpg", "phone": "+38112345678", "linkedinUrl": "https://www.linkedin.com/in/nadjakomnenic/", "icebreaker": "Honestly I'm a huge fan of your content, especially when it comes to your growth strategy as a bootstrapped startup.", "campaignName": "Saky's 1st campaign", "emailTemplateName": "onboardingEmail"}, "emitted_at": 1634230669000} -{"stream": "activities", "data": {"_id": "act_CDQdevhdThaxfu6eh", "type": "emailsOpened", "teamId": "tea_yw9qHaiihCpa8fx5t", "createdAt": "2021-10-10T21:22:40.242Z", "leadId": "lea_L5RZqF7bu7aHCPe5s", "leadEmail": "nadja+ob@lemlist.com", "campaignId": "cam_osLYq2K9dZ8Ya4Zr2", "createdBy": "usr_CTxZq8xXJBBrdgXRm", "leadFirstName": "Nadja", "leadLastName": "Komnenic", "sequenceId": "seq_je4nnaXGQ4dDy6uvS", "isFirst": false, "sequenceStep": 0, "emailTemplateId": "etp_nA35CEhmunFcPp9Q5", "extra": {"stopped": true, "leadPhone": "+38112345678", "sendUserId": "usr_CTxZq8xXJBBrdgXRm", "leadPicture": "https://app.lemlist.com/onboarding/nadja.jpg", "sendUserName": "Saky YT", "sendUserEmail": "saky_yt@outlook.com", "leadCompanyName": "lemlist"}, "firstName": "Nadja", "lastName": "Komnenic", "email": "nadja+ob@lemlist.com", "companyName": "lemlist", "picture": "https://app.lemlist.com/onboarding/nadja.jpg", "phone": "+38112345678", "linkedinUrl": "https://www.linkedin.com/in/nadjakomnenic/", "icebreaker": "Honestly I'm a huge fan of your content, especially when it comes to your growth strategy as a bootstrapped startup.", "campaignName": "Saky's 1st campaign", "emailTemplateName": "onboardingEmail"}, "emitted_at": 1634230669000} -{"stream": "activities", "data": {"_id": "act_vMSDcBFqNxsgSZxvu", "type": "emailsOpened", "teamId": "tea_yw9qHaiihCpa8fx5t", "createdAt": "2021-10-10T21:22:37.859Z", "leadId": "lea_L5RZqF7bu7aHCPe5s", "leadEmail": "nadja+ob@lemlist.com", "campaignId": "cam_osLYq2K9dZ8Ya4Zr2", "createdBy": "usr_CTxZq8xXJBBrdgXRm", "leadFirstName": "Nadja", "leadLastName": "Komnenic", "sequenceId": "seq_je4nnaXGQ4dDy6uvS", "isFirst": true, "sequenceStep": 0, "emailTemplateId": "etp_nA35CEhmunFcPp9Q5", "extra": {"leadPhone": "+38112345678", "sendUserId": "usr_CTxZq8xXJBBrdgXRm", "leadPicture": "https://app.lemlist.com/onboarding/nadja.jpg", "sendUserName": "Saky YT", "sendUserEmail": "saky_yt@outlook.com", "leadCompanyName": "lemlist"}, "firstName": "Nadja", "lastName": "Komnenic", "email": "nadja+ob@lemlist.com", "companyName": "lemlist", "picture": "https://app.lemlist.com/onboarding/nadja.jpg", "phone": "+38112345678", "linkedinUrl": "https://www.linkedin.com/in/nadjakomnenic/", "icebreaker": "Honestly I'm a huge fan of your content, especially when it comes to your growth strategy as a bootstrapped startup.", "campaignName": "Saky's 1st campaign", "emailTemplateName": "onboardingEmail"}, "emitted_at": 1634230669000} -{"stream": "activities", "data": {"_id": "act_kgSBWPrNPaHS7qE7S", "type": "emailsSent", "teamId": "tea_yw9qHaiihCpa8fx5t", "createdAt": "2021-10-10T21:22:34.427Z", "leadId": "lea_L5RZqF7bu7aHCPe5s", "leadEmail": "nadja+ob@lemlist.com", "campaignId": "cam_osLYq2K9dZ8Ya4Zr2", "createdBy": "usr_CTxZq8xXJBBrdgXRm", "leadFirstName": "Nadja", "leadLastName": "Komnenic", "sequenceId": "seq_je4nnaXGQ4dDy6uvS", "isFirst": true, "sequenceStep": 0, "emailTemplateId": "etp_nA35CEhmunFcPp9Q5", "extra": {"emailId": "eml_dkEWJr8MFGMXPbgy8", "leadPhone": "+38112345678", "messageId": "", "sendUserId": "usr_CTxZq8xXJBBrdgXRm", "leadPicture": "https://app.lemlist.com/onboarding/nadja.jpg", "sendUserName": "Saky YT", "sendUserEmail": "saky_yt@outlook.com", "leadCompanyName": "lemlist"}, "firstName": "Nadja", "lastName": "Komnenic", "email": "nadja+ob@lemlist.com", "companyName": "lemlist", "picture": "https://app.lemlist.com/onboarding/nadja.jpg", "phone": "+38112345678", "linkedinUrl": "https://www.linkedin.com/in/nadjakomnenic/", "icebreaker": "Honestly I'm a huge fan of your content, especially when it comes to your growth strategy as a bootstrapped startup.", "campaignName": "Saky's 1st campaign", "emailTemplateName": "onboardingEmail"}, "emitted_at": 1634230669000} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/streams.py b/airbyte-integrations/connectors/source-lemlist/source_lemlist/streams.py index 9dccff2449b6..cafb32347ef4 100644 --- a/airbyte-integrations/connectors/source-lemlist/source_lemlist/streams.py +++ b/airbyte-integrations/connectors/source-lemlist/source_lemlist/streams.py @@ -7,13 +7,11 @@ import requests from airbyte_cdk.sources.streams.http import HttpStream -LEMLIST_API_URL_BASE = "https://api.lemlist.com/api/" - class LemlistStream(HttpStream): """Default and max value page_size can have is 100""" - url_base = LEMLIST_API_URL_BASE + url_base = "https://api.lemlist.com/api/" primary_key = "_id" page_size = 100 initial_offset = 0