diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 1fbaef8fd9d8..4875cef67707 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -180,6 +180,7 @@ jobs: SOURCE_ZUORA_TEST_CREDS: ${{ secrets.SOURCE_ZUORA_TEST_CREDS }} SOURCE_CLOSE_COM_CREDS: ${{ secrets.SOURCE_CLOSE_COM_CREDS }} SOURCE_BAMBOO_HR_CREDS: ${{ secrets.SOURCE_BAMBOO_HR_CREDS }} + SOURCE_LINKEDIN_ADS_TEST_CREDS: ${{ secrets.SOURCE_LINKEDIN_ADS_TEST_CREDS }} SOURCE_BIGCOMMERCE_CREDS: ${{ secrets.SOURCE_BIGCOMMERCE_CREDS }} DESTINATION_DATABRICKS_CREDS: ${{ secrets.DESTINATION_DATABRICKS_CREDS }} MSSQL_SSH_KEY_TEST_CREDS: ${{ secrets.MSSQL_SSH_KEY_TEST_CREDS }} diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 092c7bfcbf08..55216adaa42e 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -175,6 +175,7 @@ jobs: SOURCE_ZUORA_TEST_CREDS: ${{ secrets.SOURCE_ZUORA_TEST_CREDS }} SOURCE_CLOSE_COM_CREDS: ${{ secrets.SOURCE_CLOSE_COM_CREDS }} SOURCE_BAMBOO_HR_CREDS: ${{ secrets.SOURCE_BAMBOO_HR_CREDS }} + SOURCE_LINKEDIN_ADS_TEST_CREDS: ${{ secrets.SOURCE_LINKEDIN_ADS_TEST_CREDS }} SOURCE_BIGCOMMERCE_CREDS: ${{ secrets.SOURCE_BIGCOMMERCE_CREDS }} DESTINATION_DATABRICKS_CREDS: ${{ secrets.DESTINATION_DATABRICKS_CREDS }} MSSQL_SSH_KEY_TEST_CREDS: ${{ secrets.MSSQL_SSH_KEY_TEST_CREDS }} diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/137ece28-5434-455c-8f34-69dc3782f451.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/137ece28-5434-455c-8f34-69dc3782f451.json new file mode 100644 index 000000000000..92b84c2b5aad --- /dev/null +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/137ece28-5434-455c-8f34-69dc3782f451.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "137ece28-5434-455c-8f34-69dc3782f451", + "name": "LinkedIn Ads", + "dockerRepository": "airbyte/source-linkedin-ads", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://docs.airbyte.io/integrations/sources/linkedin-ads" +} diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 1d068cd75788..cee91b610a7c 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -534,6 +534,11 @@ dockerRepository: airbyte/source-amazon-ads dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/amazon-ads +- sourceDefinitionId: 137ece28-5434-455c-8f34-69dc3782f451 + name: LinkedIn Ads + dockerRepository: airbyte/source-linkedin-ads + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/linkedin-ads sourceType: api - sourceDefinitionId: b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e name: MongoDb diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 471e8d0cc62d..60954759d098 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -40,6 +40,7 @@ | Intercom | [![source-intercom](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-intercom-singer%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-intercom) | | Iterable | [![source-iterable](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-iterable%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-iterable) | | Jira | [![source-jira](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-jira%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-jira) | +| LinkedIn Ads | [![source-linkedin-ads](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-linkedin-ads%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-linkedin-ads) | | Lever Hiring | [![source-lever-hiring](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-lever-hiring%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-lever-hiring) | | Looker | [![source-looker](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-looker%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-looker) | | Klaviyo | [![source-klaviyo](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-klaviyo%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-klaviyo) | diff --git a/airbyte-integrations/connectors/source-linkedin-ads/.dockerignore b/airbyte-integrations/connectors/source-linkedin-ads/.dockerignore new file mode 100644 index 000000000000..89e70f048362 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/.dockerignore @@ -0,0 +1,7 @@ +* +!Dockerfile +!Dockerfile.test +!main.py +!source_linkedin_ads +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile b/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile new file mode 100644 index 000000000000..bddce18a5155 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile @@ -0,0 +1,31 @@ +FROM python:3.7.11-alpine3.14 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade && pip install --upgrade pip + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local + +# copy payload code only +COPY main.py ./ +COPY source_linkedin_ads ./source_linkedin_ads + +# set the default Timezone, for use with dependent libraries like: datetime, pendullum, etc. +ENV TZ "UTC" +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-linkedin-ads diff --git a/airbyte-integrations/connectors/source-linkedin-ads/README.md b/airbyte-integrations/connectors/source-linkedin-ads/README.md new file mode 100644 index 000000000000..8478e7146461 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/README.md @@ -0,0 +1,137 @@ +# Linkedin Ads Source Connector + +This is the repository for the Linkedin Ads source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/linkedin-ads). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python3 -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-linkedin-ads:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/linkedin-ads) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_linkedin_ads/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source linkedin-ads test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-linkedin-ads:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew clean :airbyte-integrations:connectors:source-linkedin-ads:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-linkedin-ads:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linkedin-ads:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linkedin-ads:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-linkedin-ads:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](source-acceptance-tests.md) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +docker build . --no-cache -t airbyte/source-linkedin-ads:dev \ +&& python -m pytest -p source_acceptance_test.plugin +``` + +To run your acceptance-tests with pre-build connector docker image: +From `.venv` of the connector, run: +``` +python -m pytest -p source_acceptance_test.plugin +``` + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew clean :airbyte-integrations:connectors:source-linkedin-ads:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew clean :airbyte-integrations:connectors:source-linkedin-ads:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-linkedin-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-linkedin-ads/acceptance-test-config.yml new file mode 100644 index 000000000000..d584e7af575b --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/acceptance-test-config.yml @@ -0,0 +1,23 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/contributing-to-airbyte/building-new-connector/source-acceptance-tests) +# for more information about how to configure these tests +connector_image: airbyte/source-linkedin-ads:dev +tests: + spec: + - spec_path: "source_linkedin_ads/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-linkedin-ads/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-linkedin-ads/acceptance-test-docker.sh new file mode 100644 index 000000000000..e4d8b1cef896 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-linkedin-ads/build.gradle b/airbyte-integrations/connectors/source-linkedin-ads/build.gradle new file mode 100644 index 000000000000..fa67479f71da --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/build.gradle @@ -0,0 +1,14 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_linkedin_ads' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) + implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/__init__.py b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..d29c225e2f8c --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/abnormal_state.json @@ -0,0 +1,23 @@ +{ + "account_users": { + "lastModified": "2050-01-01" + }, + "campaign_groups": { + "lastModified": "2050-01-01" + }, + "campaigns": { + "lastModified": "2050-01-01" + }, + "creatives": { + "lastModified": "2050-01-01" + }, + "ad_direct_sponsored_contents": { + "lastModified": "2050-01-01" + }, + "ad_campaign_analytics": { + "end_date": "2050-01-01" + }, + "ad_creative_analytics": { + "end_date": "2050-01-01" + } +} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/acceptance.py new file mode 100644 index 000000000000..496a799cf8ed --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/acceptance.py @@ -0,0 +1,33 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + yield diff --git a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..3f29f785734e --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/configured_catalog.json @@ -0,0 +1,100 @@ +{ + "streams": [ + { + "stream": { + "name": "accounts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["lastModified"] + }, + "sync_mode": "incremental", + "cursor_field": ["lastModified"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "account_users", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["lastModified"] + }, + "sync_mode": "incremental", + "cursor_field": ["lastModified"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "campaign_groups", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["lastModified"] + }, + "sync_mode": "incremental", + "cursor_field": ["lastModified"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "campaigns", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["lastModified"] + }, + "sync_mode": "incremental", + "cursor_field": ["lastModified"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "creatives", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["lastModified"] + }, + "sync_mode": "incremental", + "cursor_field": ["lastModified"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_direct_sponsored_contents", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["lastModified"] + }, + "sync_mode": "incremental", + "cursor_field": ["lastModified"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_campaign_analytics", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["end_date"] + }, + "sync_mode": "incremental", + "cursor_field": ["end_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_creative_analytics", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["end_date"] + }, + "sync_mode": "incremental", + "cursor_field": ["end_date"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/invalid_config.json new file mode 100644 index 000000000000..58b20dcc7f6e --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/invalid_config.json @@ -0,0 +1,5 @@ +{ + "start_date": "2020-01-01", + "access_token": "ACCESS_TOKEN", + "account_ids": [1, 2] +} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/main.py b/airbyte-integrations/connectors/source-linkedin-ads/main.py new file mode 100644 index 000000000000..da723c1ba233 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/main.py @@ -0,0 +1,33 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_linkedin_ads import SourceLinkedinAds + +if __name__ == "__main__": + source = SourceLinkedinAds() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/requirements.txt b/airbyte-integrations/connectors/source-linkedin-ads/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-linkedin-ads/setup.py b/airbyte-integrations/connectors/source-linkedin-ads/setup.py new file mode 100644 index 000000000000..455508d2109c --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/setup.py @@ -0,0 +1,49 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk==0.1.22", + "pendulum", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "source-acceptance-test", +] + +setup( + name="source_linkedin_ads", + description="Source implementation for Linkedin Ads.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/__init__.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/__init__.py new file mode 100644 index 000000000000..041b230d536a --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/__init__.py @@ -0,0 +1,28 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from .source import SourceLinkedinAds + +__all__ = ["SourceLinkedinAds"] diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics.py new file mode 100644 index 000000000000..2c7532beb9f1 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics.py @@ -0,0 +1,205 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from collections import defaultdict +from typing import Any, Iterable, List, Mapping + +import pendulum as pdm + +from .utils import get_parent_stream_values + +# LinkedIn has a max of 20 fields per request. We make chunks by size of 17 fields +# to have the `dateRange`, `pivot`, and `pivotValue` be included as well. +FIELDS_CHUNK_SIZE = 17 +# Number of days ahead for date slices, from start date. +WINDOW_IN_DAYS = 30 +# List of adAnalyticsV2 fields available for fetch +ANALYTICS_FIELDS_V2: List = [ + "actionClicks", + "adUnitClicks", + "approximateUniqueImpressions", + "cardClicks", + "cardImpressions", + "clicks", + "commentLikes", + "comments", + "companyPageClicks", + "conversionValueInLocalCurrency", + "costInLocalCurrency", + "costInUsd", + "dateRange", + "externalWebsiteConversions", + "externalWebsitePostClickConversions", + "externalWebsitePostViewConversions", + "follows", + "fullScreenPlays", + "impressions", + "landingPageClicks", + "leadGenerationMailContactInfoShares", + "leadGenerationMailInterestedClicks", + "likes", + "oneClickLeadFormOpens", + "oneClickLeads", + "opens", + "otherEngagements", + "pivot", + "pivotValue", + "pivotValues", + "reactions", + "sends", + "shares", + "textUrlClicks", + "totalEngagements", + "videoCompletions", + "videoFirstQuartileCompletions", + "videoMidpointCompletions", + "videoStarts", + "videoThirdQuartileCompletions", + "videoViews", + "viralCardClicks", + "viralCardImpressions", + "viralClicks", + "viralCommentLikes", + "viralComments", + "viralCompanyPageClicks", + "viralExternalWebsiteConversions", + "viralExternalWebsitePostClickConversions", + "viralExternalWebsitePostViewConversions", + "viralFollows", + "viralFullScreenPlays", + "viralImpressions", + "viralLandingPageClicks", + "viralLikes", + "viralOneClickLeadFormOpens", + "viralOneClickLeads", + "viralOtherEngagements", + "viralReactions", + "viralShares", + "viralTotalEngagements", + "viralVideoCompletions", + "viralVideoFirstQuartileCompletions", + "viralVideoMidpointCompletions", + "viralVideoStarts", + "viralVideoThirdQuartileCompletions", + "viralVideoViews", +] + +# Fields that are always present in fields_set chunks +BASE_ANALLYTICS_FIELDS = ["dateRange", "pivot", "pivotValue"] + + +def chunk_analytics_fields( + fields: List = ANALYTICS_FIELDS_V2, + base_fields: List = BASE_ANALLYTICS_FIELDS, + fields_chunk_size: int = FIELDS_CHUNK_SIZE, +) -> Iterable[List]: + """ + Chunks the list of available fields into the chunks of equal size. + """ + # Make chunks + chunks = list((fields[f : f + fields_chunk_size] for f in range(0, len(fields), fields_chunk_size))) + # Make sure base_fields are within the chunks + for chunk in chunks: + for field in base_fields: + if field not in chunk: + chunk.append(field) + yield from chunks + + +def make_date_slices(start_date: str, end_date: str = None, window_in_days: int = WINDOW_IN_DAYS) -> Iterable[List]: + """ + Produces date slices from start_date to end_date (if specified), + otherwise end_date will be present time. + """ + start = pdm.parse(start_date) + end = pdm.parse(end_date) if end_date else pdm.now() + date_slices = [] + while start < end: + slice_end_date = start.add(days=window_in_days) + date_slice = { + "start.day": start.day, + "start.month": start.month, + "start.year": start.year, + "end.day": slice_end_date.day, + "end.month": slice_end_date.month, + "end.year": slice_end_date.year, + } + date_slices.append({"dateRange": date_slice}) + start = slice_end_date + yield from date_slices + + +def make_analytics_slices( + record: Mapping[str, Any], key_value_map: Mapping[str, Any], start_date: str, end_date: str = None +) -> Iterable[Mapping[str, Any]]: + """ + We drive the ability to directly pass the prepared parameters inside the stream_slice. + The output of this method is ready slices for analytics streams: + """ + # define the base_slice + base_slice = get_parent_stream_values(record, key_value_map) + # add chunked fields, date_slices to the base_slice + analytics_slices = [] + for fields_set in chunk_analytics_fields(): + base_slice["fields"] = ",".join(map(str, fields_set)) + for date_slice in make_date_slices(start_date, end_date): + base_slice.update(**date_slice) + analytics_slices.append(base_slice.copy()) + yield from analytics_slices + + +def update_analytics_params(stream_slice: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Produces the date range parameters from input stream_slice + """ + return { + # Start date range + "dateRange.start.day": stream_slice["dateRange"]["start.day"], + "dateRange.start.month": stream_slice["dateRange"]["start.month"], + "dateRange.start.year": stream_slice["dateRange"]["start.year"], + # End date range + "dateRange.end.day": stream_slice["dateRange"]["end.day"], + "dateRange.end.month": stream_slice["dateRange"]["end.month"], + "dateRange.end.year": stream_slice["dateRange"]["end.year"], + # Chunk of fields + "fields": stream_slice["fields"], + } + + +def merge_chunks(chunked_result: Iterable[Mapping[str, Any]], merge_by_key: str) -> Iterable[Mapping[str, Any]]: + """ + We need to merge the chunked API responses + into the single structure using any available unique field. + """ + # Merge the pieces together + merged = defaultdict(dict) + for chunk in chunked_result: + for item in chunk: + merged[item[merge_by_key]].update(item) + # Clean up the result by getting out the values of the merged keys + result = [] + for item in merged: + result.append(merged.get(item)) + yield from result diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/account_users.json b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/account_users.json new file mode 100644 index 000000000000..2130638974c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/account_users.json @@ -0,0 +1,22 @@ +{ + "type": ["null", "object"], + "properties": { + "account": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "lastModified": { + "type": ["null", "string"], + "format": "date-time" + }, + "role": { + "type": ["null", "string"] + }, + "user": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/accounts.json b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/accounts.json new file mode 100644 index 000000000000..d2188cfa4826 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/accounts.json @@ -0,0 +1,60 @@ +{ + "type": ["null", "object"], + "properties": { + "test": { + "type": ["null", "boolean"] + }, + "notifiedOnCreativeRejection": { + "type": ["null", "boolean"] + }, + "notifiedOnEndOfCampaign": { + "type": ["null", "boolean"] + }, + "servingStatuses": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "notifiedOnCampaignOptimization": { + "type": ["null", "boolean"] + }, + "type": { + "type": ["null", "string"] + }, + "version": { + "type": ["null", "object"], + "properties": { + "versionTag": { + "type": ["null", "string"] + } + } + }, + "reference": { + "type": ["null", "string"] + }, + "notifiedOnCreativeApproval": { + "type": ["null", "boolean"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "lastModified": { + "type": ["null", "string"], + "format": "date-time" + }, + "name": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/ad_campaign_analytics.json b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/ad_campaign_analytics.json new file mode 100644 index 000000000000..24d28fbd5d42 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/ad_campaign_analytics.json @@ -0,0 +1,196 @@ +{ + "type": ["null", "object"], + "properties": { + "actionClicks": { + "type": ["null", "number"] + }, + "adUnitClicks": { + "type": ["null", "number"] + }, + "clicks": { + "type": ["null", "number"] + }, + "comments": { + "type": ["null", "number"] + }, + "companyPageClicks": { + "type": ["null", "number"] + }, + "conversionValueInLocalCurrency": { + "type": ["null", "string"] + }, + "costInLocalCurrency": { + "type": ["null", "string"] + }, + "costInUsd": { + "type": ["null", "string"] + }, + "start_date": { + "type": ["null", "string"], + "format": "date" + }, + "end_date": { + "type": ["null", "string"], + "format": "date" + }, + "externalWebsiteConversions": { + "type": ["null", "number"] + }, + "externalWebsitePostClickConversions": { + "type": ["null", "number"] + }, + "externalWebsitePostViewConversions": { + "type": ["null", "number"] + }, + "follows": { + "type": ["null", "number"] + }, + "fullScreenPlays": { + "type": ["null", "number"] + }, + "impressions": { + "type": ["null", "number"] + }, + "landingPageClicks": { + "type": ["null", "number"] + }, + "leadGenerationMailContactInfoShares": { + "type": ["null", "number"] + }, + "leadGenerationMailInterestedClicks": { + "type": ["null", "number"] + }, + "likes": { + "type": ["null", "number"] + }, + "oneClickLeadFormOpens": { + "type": ["null", "number"] + }, + "oneClickLeads": { + "type": ["null", "number"] + }, + "opens": { + "type": ["null", "number"] + }, + "otherEngagements": { + "type": ["null", "number"] + }, + "pivot": { + "type": ["null", "string"] + }, + "pivotValue": { + "type": ["null", "string"] + }, + "pivotValues": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "reactions": { + "type": ["null", "number"] + }, + "sends": { + "type": ["null", "number"] + }, + "shares": { + "type": ["null", "number"] + }, + "textUrlClicks": { + "type": ["null", "number"] + }, + "totalEngagements": { + "type": ["null", "number"] + }, + "videoCompletions": { + "type": ["null", "number"] + }, + "videoFirstQuartileCompletions": { + "type": ["null", "number"] + }, + "videoMidpointCompletions": { + "type": ["null", "number"] + }, + "videoStarts": { + "type": ["null", "number"] + }, + "videoThirdQuartileCompletions": { + "type": ["null", "number"] + }, + "videoViews": { + "type": ["null", "number"] + }, + "viralClicks": { + "type": ["null", "number"] + }, + "viralComments": { + "type": ["null", "number"] + }, + "viralCommentLikes": { + "type": ["null", "number"] + }, + "viralCompanyPageClicks": { + "type": ["null", "number"] + }, + "viralExternalWebsiteConversions": { + "type": ["null", "number"] + }, + "viralExternalWebsitePostClickConversions": { + "type": ["null", "number"] + }, + "viralExternalWebsitePostViewConversions": { + "type": ["null", "number"] + }, + "viralFollows": { + "type": ["null", "number"] + }, + "viralFullScreenPlays": { + "type": ["null", "number"] + }, + "viralImpressions": { + "type": ["null", "number"] + }, + "viralLandingPageClicks": { + "type": ["null", "number"] + }, + "viralLikes": { + "type": ["null", "number"] + }, + "viralOneClickLeadFormOpens": { + "type": ["null", "number"] + }, + "viralOneClickLeads": { + "type": ["null", "number"] + }, + "viralOtherEngagements": { + "type": ["null", "number"] + }, + "viralReactions": { + "type": ["null", "number"] + }, + "viralShares": { + "type": ["null", "number"] + }, + "viralTotalEngagements": { + "type": ["null", "number"] + }, + "viralVideoCompletions": { + "type": ["null", "number"] + }, + "viralVideoFirstQuartileCompletions": { + "type": ["null", "number"] + }, + "viralVideoMidpointCompletions": { + "type": ["null", "number"] + }, + "viralVideoStarts": { + "type": ["null", "number"] + }, + "viralVideoThirdQuartileCompletions": { + "type": ["null", "number"] + }, + "viralVideoViews": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/ad_creative_analytics.json b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/ad_creative_analytics.json new file mode 100644 index 000000000000..24d28fbd5d42 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/ad_creative_analytics.json @@ -0,0 +1,196 @@ +{ + "type": ["null", "object"], + "properties": { + "actionClicks": { + "type": ["null", "number"] + }, + "adUnitClicks": { + "type": ["null", "number"] + }, + "clicks": { + "type": ["null", "number"] + }, + "comments": { + "type": ["null", "number"] + }, + "companyPageClicks": { + "type": ["null", "number"] + }, + "conversionValueInLocalCurrency": { + "type": ["null", "string"] + }, + "costInLocalCurrency": { + "type": ["null", "string"] + }, + "costInUsd": { + "type": ["null", "string"] + }, + "start_date": { + "type": ["null", "string"], + "format": "date" + }, + "end_date": { + "type": ["null", "string"], + "format": "date" + }, + "externalWebsiteConversions": { + "type": ["null", "number"] + }, + "externalWebsitePostClickConversions": { + "type": ["null", "number"] + }, + "externalWebsitePostViewConversions": { + "type": ["null", "number"] + }, + "follows": { + "type": ["null", "number"] + }, + "fullScreenPlays": { + "type": ["null", "number"] + }, + "impressions": { + "type": ["null", "number"] + }, + "landingPageClicks": { + "type": ["null", "number"] + }, + "leadGenerationMailContactInfoShares": { + "type": ["null", "number"] + }, + "leadGenerationMailInterestedClicks": { + "type": ["null", "number"] + }, + "likes": { + "type": ["null", "number"] + }, + "oneClickLeadFormOpens": { + "type": ["null", "number"] + }, + "oneClickLeads": { + "type": ["null", "number"] + }, + "opens": { + "type": ["null", "number"] + }, + "otherEngagements": { + "type": ["null", "number"] + }, + "pivot": { + "type": ["null", "string"] + }, + "pivotValue": { + "type": ["null", "string"] + }, + "pivotValues": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "reactions": { + "type": ["null", "number"] + }, + "sends": { + "type": ["null", "number"] + }, + "shares": { + "type": ["null", "number"] + }, + "textUrlClicks": { + "type": ["null", "number"] + }, + "totalEngagements": { + "type": ["null", "number"] + }, + "videoCompletions": { + "type": ["null", "number"] + }, + "videoFirstQuartileCompletions": { + "type": ["null", "number"] + }, + "videoMidpointCompletions": { + "type": ["null", "number"] + }, + "videoStarts": { + "type": ["null", "number"] + }, + "videoThirdQuartileCompletions": { + "type": ["null", "number"] + }, + "videoViews": { + "type": ["null", "number"] + }, + "viralClicks": { + "type": ["null", "number"] + }, + "viralComments": { + "type": ["null", "number"] + }, + "viralCommentLikes": { + "type": ["null", "number"] + }, + "viralCompanyPageClicks": { + "type": ["null", "number"] + }, + "viralExternalWebsiteConversions": { + "type": ["null", "number"] + }, + "viralExternalWebsitePostClickConversions": { + "type": ["null", "number"] + }, + "viralExternalWebsitePostViewConversions": { + "type": ["null", "number"] + }, + "viralFollows": { + "type": ["null", "number"] + }, + "viralFullScreenPlays": { + "type": ["null", "number"] + }, + "viralImpressions": { + "type": ["null", "number"] + }, + "viralLandingPageClicks": { + "type": ["null", "number"] + }, + "viralLikes": { + "type": ["null", "number"] + }, + "viralOneClickLeadFormOpens": { + "type": ["null", "number"] + }, + "viralOneClickLeads": { + "type": ["null", "number"] + }, + "viralOtherEngagements": { + "type": ["null", "number"] + }, + "viralReactions": { + "type": ["null", "number"] + }, + "viralShares": { + "type": ["null", "number"] + }, + "viralTotalEngagements": { + "type": ["null", "number"] + }, + "viralVideoCompletions": { + "type": ["null", "number"] + }, + "viralVideoFirstQuartileCompletions": { + "type": ["null", "number"] + }, + "viralVideoMidpointCompletions": { + "type": ["null", "number"] + }, + "viralVideoStarts": { + "type": ["null", "number"] + }, + "viralVideoThirdQuartileCompletions": { + "type": ["null", "number"] + }, + "viralVideoViews": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/ad_direct_sponsored_contents.json b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/ad_direct_sponsored_contents.json new file mode 100644 index 000000000000..fd1029de15cd --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/ad_direct_sponsored_contents.json @@ -0,0 +1,31 @@ +{ + "type": ["null", "object"], + "properties": { + "owner": { + "type": ["null", "string"] + }, + "contentReference": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "account": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "lastModified": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/campaign_groups.json b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/campaign_groups.json new file mode 100644 index 000000000000..f637f99b56a2 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/campaign_groups.json @@ -0,0 +1,54 @@ +{ + "type": ["null", "object"], + "properties": { + "runSchedule": { + "type": ["null", "object"], + "properties": { + "start": { + "type": ["null", "integer"] + }, + "end": { + "type": ["null", "integer"] + } + } + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "lastModified": { + "type": ["null", "string"], + "format": "date-time" + }, + "name": { + "type": ["null", "string"] + }, + "test": { + "type": ["null", "boolean"] + }, + "servingStatuses": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "allowedCampaignTypes": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "backfilled": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer"] + }, + "account": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/campaigns.json b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/campaigns.json new file mode 100644 index 000000000000..e223d6ba0807 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/campaigns.json @@ -0,0 +1,168 @@ +{ + "type": ["null", "object"], + "properties": { + "targetingCriteria": { + "type": ["null", "object"], + "properties": { + "include": { + "type": ["null", "object"], + "properties": { + "and": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "values": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + } + } + } + }, + "exclude": { + "type": ["null", "object"], + "properties": { + "or": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "values": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + } + } + } + } + } + }, + "servingStatuses": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "type": { + "type": ["null", "string"] + }, + "locale": { + "type": ["null", "object"], + "properties": { + "country": { + "type": ["null", "string"] + }, + "language": { + "type": ["null", "string"] + } + } + }, + "version": { + "type": ["null", "object"], + "properties": { + "versionTag": { + "type": ["null", "string"] + } + } + }, + "associatedEntity": { + "type": ["null", "string"] + }, + "runSchedule": { + "type": ["null", "object"], + "properties": { + "start": { + "type": ["null", "integer"] + }, + "end": { + "type": ["null", "integer"] + } + } + }, + "optimizationTargetType": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "lastModified": { + "type": ["null", "string"], + "format": "date-time" + }, + "campaignGroup": { + "type": ["null", "string"] + }, + "dailyBudget": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currencyCode": { + "type": ["null", "string"] + } + } + }, + "unitCost": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currencyCode": { + "type": ["null", "string"] + } + } + }, + "creativeSelection": { + "type": ["null", "string"] + }, + "costType": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "offsiteDeliveryEnabled": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer"] + }, + "audienceExpansionEnabled": { + "type": ["null", "boolean"] + }, + "account": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "storyDeliveryEnabled": { + "type": ["null", "boolean"] + }, + "pacingStrategy": { + "type": ["null", "string"] + }, + "format": { + "type": ["null", "string"] + }, + "objectiveType": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/creatives.json b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/creatives.json new file mode 100644 index 000000000000..62559878dd8d --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/creatives.json @@ -0,0 +1,96 @@ +{ + "type": ["null", "object"], + "properties": { + "campaign": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "lastModified": { + "type": ["null", "string"], + "format": "date-time" + }, + "id": { + "type": ["null", "integer"] + }, + "reference": { + "type": ["null", "string"] + }, + "review": { + "type": ["null", "object"], + "properties": { + "reviewStatus": { + "type": ["null", "string"] + } + } + }, + "status": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "variables": { + "type": ["null", "object"], + "properties": { + "click_uri": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "values": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "key": { + "type": ["null", "string"] + }, + "value": { + "anyOf": [ + { + "type": ["null", "string"] + }, + { + "type": ["null", "boolean"] + }, + { + "type": ["null", "number"] + }, + { + "type": ["null", "integer"] + }, + { + "type": ["null", "object"], + "additionalProperties": true + } + ] + } + } + } + } + } + }, + "version": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "versionTag": { + "type": ["null", "string"] + } + } + }, + "test": { + "type": ["null", "boolean"] + }, + "servingStatuses": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py new file mode 100644 index 000000000000..81381d0a78fb --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py @@ -0,0 +1,360 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from abc import ABC, abstractproperty +from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from urllib.parse import urlencode + +import requests +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + +from .analytics import make_analytics_slices, merge_chunks, update_analytics_params +from .utils import get_parent_stream_values, transform_data + + +class LinkedinAdsStream(HttpStream, ABC): + """ + Basic class provides base functionality for all streams. + """ + + url_base = "https://api.linkedin.com/v2/" + primary_key = "id" + records_limit = 500 + + def __init__(self, config: Dict): + super().__init__(authenticator=config.get("authenticator")) + self.config = config + + @property + def accounts(self): + """ Property to return the list of the user Account Ids from input """ + return ",".join(map(str, self.config.get("account_ids"))) + + def path(self, **kwargs) -> str: + """ Returns the API endpoint path for stream, from `endpoint` class attribute. """ + return self.endpoint + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + To paginate through results, begin with a start value of 0 and a count value of N. + To get the next page, set start value to N, while the count value stays the same. + We have reached the end of the dataset when the response contains fewer elements than the `count` parameter request. + https://docs.microsoft.com/en-us/linkedin/shared/api-guide/concepts/pagination?context=linkedin/marketing/context + """ + parsed_response = response.json() + if len(parsed_response.get("elements")) < self.records_limit: + return None + return {"start": parsed_response.get("paging").get("start") + self.records_limit} + + def request_params( + self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs + ) -> MutableMapping[str, Any]: + params = {"count": self.records_limit, "q": "search"} + if next_page_token: + params.update(**next_page_token) + return params + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + We need to get out the nested complex data structures for further normalisation, so the transform_data method is applied. + """ + yield from transform_data(response.json().get("elements")) + + +class Accounts(LinkedinAdsStream): + """ + Get Accounts data. More info about LinkedIn Ads / Accounts: + https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads/account-structure/create-and-manage-accounts?tabs=http + """ + + endpoint = "adAccountsV2" + + def request_headers(self, stream_state: Mapping[str, Any], **kwargs) -> Mapping[str, Any]: + """ + If account_ids are specified as user's input from configuration, + we must use MODIFIED header: {'X-RestLi-Protocol-Version': '2.0.0'} + """ + return {"X-RestLi-Protocol-Version": "2.0.0"} if self.accounts else {} + + def request_params(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + """ + Override request_params() to have the ability to accept the specific account_ids from user's configuration. + If we have list of account_ids, we need to make sure that the request_params are encoded correctly, + We will get HTTP Error 500, if we use standard requests.urlencode methods to parse parameters, + so the urlencode(..., safe=":(),") is used instead, to keep the values as they are. + """ + params = super().request_params(stream_state=stream_state, **kwargs) + if self.accounts: + params["search"] = f"(id:(values:List({self.accounts})))" + return urlencode(params, safe=":(),") + return params + + +class IncrementalLinkedinAdsStream(LinkedinAdsStream): + + cursor_field = "lastModified" + + @property + def primary_slice_key(self) -> str: + """ + Define the main slice_key from `slice_key_value_map`. Always the first element. + EXAMPLE: + in : {"k1": "v1", "k2": "v2", ...} + out : "k1" + """ + return list(self.parent_values_map.keys())[0] + + @abstractproperty + def parent_stream(self) -> object: + """ Defines the parrent stream for slicing, the class object should be provided. """ + + @property + def state_checkpoint_interval(self) -> Optional[int]: + """ Define the checkpoint from the records output size. """ + return super().records_limit + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + current_stream_state = {self.cursor_field: self.config.get("start_date")} if not current_stream_state else current_stream_state + return {self.cursor_field: max(latest_record.get(self.cursor_field), current_stream_state.get(self.cursor_field))} + + +class LinkedInAdsStreamSlicing(IncrementalLinkedinAdsStream): + """ + This class stands for provide stream slicing for other dependent streams. + :: `parent_stream` - the reference to the parent stream class, + by default it's referenced to the Accounts stream class, as far as majority of streams are using it. + :: `parent_values_map` - key_value map for stream slices in a format: {: } + :: `search_param` - the query param to pass with request_params + :: `search_param_value` - the value for `search_param` to pass with request_params + """ + + parent_stream = Accounts + parent_values_map = {"account_id": "id"} + # define default additional request params + search_param = "search.account.values[0]" + search_param_value = "urn:li:sponsoredAccount:" + + def request_params(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=stream_state, **kwargs) + params[self.search_param] = f"{self.search_param_value}{stream_slice.get(self.primary_slice_key)}" + return params + + def filter_records_newer_than_state(self, stream_state: Mapping[str, Any] = None, records_slice: Mapping[str, Any] = None) -> Iterable: + """ For the streams that provide the cursor_field `lastModified`, we filter out the old records. """ + if stream_state: + for record in records_slice: + if record[self.cursor_field] >= stream_state.get(self.cursor_field): + yield record + else: + yield from records_slice + + def read_records( + self, stream_state: Mapping[str, Any] = None, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs + ) -> Iterable[Mapping[str, Any]]: + stream_state = stream_state or {} + parent_stream = self.parent_stream(config=self.config) + for record in parent_stream.read_records(**kwargs): + child_stream_slice = super().read_records(stream_slice=get_parent_stream_values(record, self.parent_values_map), **kwargs) + yield from self.filter_records_newer_than_state(stream_state=stream_state, records_slice=child_stream_slice) + + +class AccountUsers(LinkedInAdsStreamSlicing): + """ + Get AccountUsers data using `account_id` slicing. More info about LinkedIn Ads / AccountUsers: + https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads/account-structure/create-and-manage-account-users?tabs=http + """ + + endpoint = "adAccountUsersV2" + # Account_users stream doesn't have `id` property, so the "account" is used instead. + primary_key = "account" + search_param = "accounts" + + def request_params(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=stream_state, **kwargs) + params["q"] = self.search_param + return params + + +class CampaignGroups(LinkedInAdsStreamSlicing): + """ + Get CampaignGroups data using `account_id` slicing. + More info about LinkedIn Ads / CampaignGroups: + https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads/account-structure/create-and-manage-campaign-groups?tabs=http + """ + + endpoint = "adCampaignGroupsV2" + + +class Campaigns(LinkedInAdsStreamSlicing): + """ + Get Campaigns data using `account_id` slicing. + More info about LinkedIn Ads / Campaigns: + https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads/account-structure/create-and-manage-campaigns?tabs=http + """ + + endpoint = "adCampaignsV2" + + +class Creatives(LinkedInAdsStreamSlicing): + """ + Get Creatives data using `campaign_id` slicing. + More info about LinkedIn Ads / Creatives: + https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads/account-structure/create-and-manage-creatives?tabs=http + """ + + endpoint = "adCreativesV2" + parent_stream = Campaigns + parent_values_map = {"campaign_id": "id"} + search_param = "search.campaign.values[0]" + search_param_value = "urn:li:sponsoredCampaign:" + + +class AdDirectSponsoredContents(LinkedInAdsStreamSlicing): + """ + Get AdDirectSponsoredContents data using `account_id` slicing. + More info about LinkedIn Ads / adDirectSponsoredContents: + https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads/advertising-targeting/create-and-manage-video?tabs=http#finders + """ + + endpoint = "adDirectSponsoredContents" + # AdDirectSponsoredContents stream doesn't have `id` property, so the "account" is used instead. + primary_key = "account" + parent_values_map = {"account_id": "id", "reference_id": "reference"} + search_param = "account" + + def request_params(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, **kwargs) + params["owner"] = stream_slice.get("reference_id") + params["q"] = self.search_param + return params + + +class LinkedInAdsAnalyticsStream(IncrementalLinkedinAdsStream): + """ + AdAnalytics Streams more info: + https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?tabs=curl#ad-analytics + """ + + endpoint = "adAnalyticsV2" + # For Analytics streams the primary_key is the entity of the pivot [Campaign URN, Creative URN, etc] + `end_date` + primary_key = ["pivotValue", "end_date"] + cursor_field = "end_date" + + @property + def base_analytics_params(self) -> MutableMapping[str, Any]: + """ Define the base parameters for analytics streams """ + return {"q": "analytics", "pivot": self.pivot_by, "timeGranularity": "DAILY"} + + def request_params(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + params = self.base_analytics_params + params[self.search_param] = f"{self.search_param_value}{stream_slice.get(self.primary_slice_key)}" + params.update(**update_analytics_params(stream_slice)) + return params + + def read_records( + self, stream_state: Mapping[str, Any] = None, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs + ) -> Iterable[Mapping[str, Any]]: + stream_state = stream_state or {self.cursor_field: self.config.get("start_date")} + parent_stream = self.parent_stream(config=self.config) + for record in parent_stream.read_records(**kwargs): + result_chunks = [] + for analytics_slice in make_analytics_slices(record, self.parent_values_map, stream_state.get(self.cursor_field)): + child_stream_slice = super().read_records(stream_slice=analytics_slice, **kwargs) + result_chunks.append(child_stream_slice) + yield from merge_chunks(result_chunks, self.cursor_field) + + +class AdCampaignAnalytics(LinkedInAdsAnalyticsStream): + """ + Campaing Analytics stream. + See the AnalyticsStreamMixin class for more information. + """ + + parent_stream = Campaigns + parent_values_map = {"campaign_id": "id"} + search_param = "campaigns[0]" + search_param_value = "urn:li:sponsoredCampaign:" + pivot_by = "CAMPAIGN" + + +class AdCreativeAnalytics(LinkedInAdsAnalyticsStream): + """ + Creative Analytics stream. + See the AnalyticsStreamMixin class for more information. + """ + + parent_stream = Creatives + parent_values_map = {"creative_id": "id"} + search_param = "creatives[0]" + search_param_value = "urn:li:sponsoredCreative:" + pivot_by = "CREATIVE" + + +class SourceLinkedinAds(AbstractSource): + """ + Abstract Source inheritance, provides: + - implementation for `check` connector's connectivity + - implementation to call each stream with it's input parameters. + """ + + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: + """ + Testing connection availability for the connector. + :: for this check method the Customer must have the "r_liteprofile" scope enabled. + :: more info: https://docs.microsoft.com/linkedin/consumer/integrations/self-serve/sign-in-with-linkedin + """ + + header = TokenAuthenticator(token=config["access_token"]).get_auth_header() + profile_url = "https://api.linkedin.com/v2/me" + + try: + response = requests.get(url=profile_url, headers=header) + response.raise_for_status() + return True, None + except requests.exceptions.RequestException as e: + return False, f"{e}, {response.json().get('message')}" + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + Mapping a input config of the user input configuration as defined in the connector spec. + Passing config to the streams. + """ + + config["authenticator"] = TokenAuthenticator(token=config["access_token"]) + + return [ + Accounts(config), + AccountUsers(config), + AdCampaignAnalytics(config), + AdCreativeAnalytics(config), + AdDirectSponsoredContents(config), + CampaignGroups(config), + Campaigns(config), + Creatives(config), + ] diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/spec.json b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/spec.json new file mode 100644 index 000000000000..491664b45a1c --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/spec.json @@ -0,0 +1,34 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/linkedin-ads", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Linkedin Ads Spec", + "type": "object", + "required": ["start_date", "access_token"], + "additionalProperties": false, + "properties": { + "start_date": { + "type": "string", + "title": "Start Date", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "description": "Date in the format 2020-09-17. Any data before this date will not be replicated.", + "examples": ["2021-05-17"] + }, + "access_token": { + "type": "string", + "title": "Access Token", + "description": "The token value ganerated using Auth Code", + "airbyte_secret": true + }, + "account_ids": { + "title": "Account IDs", + "type": "array", + "description": "Specify the Account IDs separated by space, from which to pull the data. Leave empty to pull from all associated accounts.", + "items": { + "type": "integer" + }, + "default": [] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py new file mode 100644 index 000000000000..310df7658af1 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py @@ -0,0 +1,331 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from typing import Any, Dict, Iterable, List, Mapping + +import pendulum as pdm + + +def get_parent_stream_values(record: Dict, key_value_map: Dict) -> Dict: + """ + Outputs the Dict with key:value slices for the stream. + :: EXAMPLE: + Input: + records = [{dict}, {dict}, ...], + key_value_map = {: } + + Output: + { + : records..value, + } + """ + result = {} + for key in key_value_map: + value = record.get(key_value_map[key]) + if value: + result[key] = value + return result + + +def transform_change_audit_stamps( + record: Dict, dict_key: str = "changeAuditStamps", props: List = ["created", "lastModified"], fields: List = ["time"] +) -> Mapping[str, Any]: + + """ + :: EXAMPLE `changeAuditStamps` input structure: + { + "changeAuditStamps": { + "created": {"time": 1629581275000}, + "lastModified": {"time": 1629664544760} + } + } + + :: EXAMPLE output: + { + "created": "2021-08-21 21:27:55", + "lastModified": "2021-08-22 20:35:44" + } + """ + + target_dict: Dict = record.get(dict_key) + for prop in props: + # Update dict with flatten key:value + for field in fields: + record[prop] = pdm.from_timestamp(target_dict.get(prop).get(field) / 1000).to_datetime_string() + record.pop(dict_key) + + return record + + +def date_str_from_date_range(record: Dict, prefix: str) -> str: + """ + Makes the ISO8601 format date string from the input . + + EXAMPLE: + Input: record + { + "start.year": 2021, "start.month": 8, "start.day": 1, + "end.year": 2021, "end.month": 9, "end.day": 31 + } + + EXAMPLE output: + With `prefix` = "start" + str: "2021-08-13", + + With `prefix` = "end" + str: "2021-09-31", + """ + + year = record.get(f"{prefix}.year") + month = record.get(f"{prefix}.month") + day = record.get(f"{prefix}.day") + return pdm.date(year, month, day).to_date_string() + + +def transform_date_range( + record: Dict, + dict_key: str = "dateRange", + props: List = ["start", "end"], + fields: List = ["year", "month", "day"], +) -> Mapping[str, Any]: + + """ + :: EXAMPLE `dateRange` input structure in Analytics streams: + { + "dateRange": { + "start": {"month": 8, "day": 13, "year": 2021}, + "end": {"month": 8, "day": 13, "year": 2021} + } + } + :: EXAMPLE output: + { + "start_date": "2021-08-13", + "end_date": "2021-08-13" + } + """ + # define list of tmp keys for cleanup. + keys_to_remove = [dict_key, "start.day", "start.month", "start.year", "end.day", "end.month", "end.year", "start", "end"] + + target_dict: Dict = record.get(dict_key) + for prop in props: + # Update dict with flatten key:value + for field in fields: + record.update(**{f"{prop}.{field}": target_dict.get(prop).get(field)}) + # We build `start_date` & `end_date` fields from nested structure. + record.update(**{"start_date": date_str_from_date_range(record, "start"), "end_date": date_str_from_date_range(record, "end")}) + # Cleanup tmp fields & nested used parts + for key in keys_to_remove: + if key in record: + record.pop(key) + return record + + +def transform_targeting_criteria( + record: Dict, + dict_key: str = "targetingCriteria", +) -> Mapping[str, Any]: + + """ + :: EXAMPLE `targetingCriteria` input structure: + { + "targetingCriteria": { + "include": { + "and": [ + { + "or": { + "urn:li:adTargetingFacet:titles": [ + "urn:li:title:100", + "urn:li:title:10326", + "urn:li:title:10457", + "urn:li:title:10738", + "urn:li:title:10966", + "urn:li:title:11349", + "urn:li:title:1159", + ] + } + }, + {"or": {"urn:li:adTargetingFacet:locations": ["urn:li:geo:103644278"]}}, + {"or": {"urn:li:adTargetingFacet:interfaceLocales": ["urn:li:locale:en_US"]}}, + ] + }, + "exclude": { + "or": { + "urn:li:adTargetingFacet:facet_Key1": [ + "facet_test1", + "facet_test2", + ], + "urn:li:adTargetingFacet:facet_Key2": [ + "facet_test3", + "facet_test4", + ], + } + } + } + + :: EXAMPLE output: + { + "targetingCriteria": { + "include": { + "and": [ + { + "type": "urn:li:adTargetingFacet:titles", + "values": [ + "urn:li:title:100", + "urn:li:title:10326", + "urn:li:title:10457", + "urn:li:title:10738", + "urn:li:title:10966", + "urn:li:title:11349", + "urn:li:title:1159", + ], + }, + { + "type": "urn:li:adTargetingFacet:locations", + "values": ["urn:li:geo:103644278"], + }, + { + "type": "urn:li:adTargetingFacet:interfaceLocales", + "values": ["urn:li:locale:en_US"], + }, + ] + }, + "exclude": { + "or": [ + { + "type": "urn:li:adTargetingFacet:facet_Key1", + "values": ["facet_test1", "facet_test2"], + }, + { + "type": "urn:li:adTargetingFacet:facet_Key2", + "values": ["facet_test3", "facet_test4"], + }, + ] + }, + } + + """ + targeting_criteria = record.get(dict_key) + # transform `include` + if "include" in targeting_criteria: + and_list = targeting_criteria.get("include").get("and") + for id, and_criteria in enumerate(and_list): + or_dict = and_criteria.get("or") + for key, value in or_dict.items(): + values = [] + if isinstance(value, list): + if isinstance(value[0], str): + values = value + elif isinstance(value[0], dict): + for v in value: + values.append(v) + elif isinstance(key, dict): + values.append(key) + # Replace the 'or' with {type:value} + record["targetingCriteria"]["include"]["and"][id]["type"] = key + record["targetingCriteria"]["include"]["and"][id]["values"] = values + record["targetingCriteria"]["include"]["and"][id].pop("or") + + # transform `exclude` if present + if "exclude" in targeting_criteria: + or_dict = targeting_criteria.get("exclude").get("or") + updated_exclude = {"or": []} + for key, value in or_dict.items(): + values = [] + if isinstance(value, list): + if isinstance(value[0], str): + values = value + elif isinstance(value[0], dict): + for v in value: + value.append(v) + elif isinstance(value, dict): + value.append(value) + updated_exclude["or"].append({"type": key, "values": values}) + record["targetingCriteria"]["exclude"] = updated_exclude + + return record + + +def transform_variables( + record: Dict, + dict_key: str = "variables", +) -> Mapping[str, Any]: + + """ + :: EXAMPLE `variables` input: + { + "variables": { + "data": { + "com.linkedin.ads.SponsoredUpdateCreativeVariables": { + "activity": "urn:li:activity:1234", + "directSponsoredContent": 0, + "share": "urn:li:share:1234", + } + } + } + } + + :: EXAMPLE output: + { + "variables": { + "type": "com.linkedin.ads.SponsoredUpdateCreativeVariables", + "values": [ + {"key": "activity", "value": "urn:li:activity:1234"}, + {"key": "directSponsoredContent", "value": 0}, + {"key": "share", "value": "urn:li:share:1234"}, + ], + } + } + """ + + variables = record.get(dict_key).get("data") + for key, params in variables.items(): + record["variables"]["type"] = key + record["variables"]["values"] = [] + for key, param in params.items(): + record["variables"]["values"].append({"key": key, "value": param}) + # Clean the nested structure + record["variables"].pop("data") + return record + + +def transform_data(records: List) -> Iterable[Mapping]: + """ + We need to transform the nested complex data structures into simple key:value pair, + to be properly normalised in the destination. + """ + for record in records: + + if "changeAuditStamps" in record: + record = transform_change_audit_stamps(record) + + if "dateRange" in record: + record = transform_date_range(record) + + if "targetingCriteria" in record: + record = transform_targeting_criteria(record) + + if "variables" in record: + record = transform_variables(record) + + yield record diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/samples/test_data_for_analytics.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/samples/test_data_for_analytics.py new file mode 100644 index 000000000000..7d26b7096161 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/samples/test_data_for_analytics.py @@ -0,0 +1,239 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from typing import Dict, List + +""" +This is the example of input record for the test_make_analytics_slices. +""" +test_input_record: Dict = { + "id": 123, + "audienceExpansionEnabled": True, + "test": False, + "format": "STANDARD_UPDATE", + "servingStatuses": ["CAMPAIGN_GROUP_TOTAL_BUDGET_HOLD"], + "version": {"versionTag": "2"}, + "objectiveType": "TEST_TEST", + "associatedEntity": "urn:li:organization:456", + "offsitePreferences": { + "iabCategories": {"exclude": []}, + "publisherRestrictionFiles": {"exclude": []}, + }, + "campaignGroup": "urn:li:sponsoredCampaignGroup:1234567", + "account": "urn:li:sponsoredAccount:123456", + "status": "ACTIVE", + "created": "2021-08-06 06:03:52", + "lastModified": "2021-08-06 06:09:04", +} + + +""" +This is the expected output from the `make_analytics_slices` method. +VALID PARAMETERS FOR THE OUTPUT ARE: +: TEST_KEY_VALUE_MAP = {"campaign_id": "id"} +: TEST_START_DATE = "2021-08-01" +: TEST_END_DATE = "2021-09-30" + +Change the input parameters inside of test_make_analytics_slices.py unit test. +Make sure for valid KEY_VALUE_MAP references inside of the `test_input_record` +""" +test_output_slices: List = [ + { + "camp_id": 123, + "fields": "actionClicks,adUnitClicks,approximateUniqueImpressions,cardClicks,cardImpressions,clicks,commentLikes,comments,companyPageClicks,conversionValueInLocalCurrency,costInLocalCurrency,costInUsd,dateRange,externalWebsiteConversions,externalWebsitePostClickConversions,externalWebsitePostViewConversions,follows,pivot,pivotValue", + "dateRange": { + "start.day": 1, + "start.month": 8, + "start.year": 2021, + "end.day": 31, + "end.month": 8, + "end.year": 2021, + }, + }, + { + "camp_id": 123, + "fields": "actionClicks,adUnitClicks,approximateUniqueImpressions,cardClicks,cardImpressions,clicks,commentLikes,comments,companyPageClicks,conversionValueInLocalCurrency,costInLocalCurrency,costInUsd,dateRange,externalWebsiteConversions,externalWebsitePostClickConversions,externalWebsitePostViewConversions,follows,pivot,pivotValue", + "dateRange": { + "start.day": 31, + "start.month": 8, + "start.year": 2021, + "end.day": 30, + "end.month": 9, + "end.year": 2021, + }, + }, + { + "camp_id": 123, + "fields": "fullScreenPlays,impressions,landingPageClicks,leadGenerationMailContactInfoShares,leadGenerationMailInterestedClicks,likes,oneClickLeadFormOpens,oneClickLeads,opens,otherEngagements,pivot,pivotValue,pivotValues,reactions,sends,shares,textUrlClicks,dateRange", + "dateRange": { + "start.day": 1, + "start.month": 8, + "start.year": 2021, + "end.day": 31, + "end.month": 8, + "end.year": 2021, + }, + }, + { + "camp_id": 123, + "fields": "fullScreenPlays,impressions,landingPageClicks,leadGenerationMailContactInfoShares,leadGenerationMailInterestedClicks,likes,oneClickLeadFormOpens,oneClickLeads,opens,otherEngagements,pivot,pivotValue,pivotValues,reactions,sends,shares,textUrlClicks,dateRange", + "dateRange": { + "start.day": 31, + "start.month": 8, + "start.year": 2021, + "end.day": 30, + "end.month": 9, + "end.year": 2021, + }, + }, + { + "camp_id": 123, + "fields": "totalEngagements,videoCompletions,videoFirstQuartileCompletions,videoMidpointCompletions,videoStarts,videoThirdQuartileCompletions,videoViews,viralCardClicks,viralCardImpressions,viralClicks,viralCommentLikes,viralComments,viralCompanyPageClicks,viralExternalWebsiteConversions,viralExternalWebsitePostClickConversions,viralExternalWebsitePostViewConversions,viralFollows,dateRange,pivot,pivotValue", + "dateRange": { + "start.day": 1, + "start.month": 8, + "start.year": 2021, + "end.day": 31, + "end.month": 8, + "end.year": 2021, + }, + }, + { + "camp_id": 123, + "fields": "totalEngagements,videoCompletions,videoFirstQuartileCompletions,videoMidpointCompletions,videoStarts,videoThirdQuartileCompletions,videoViews,viralCardClicks,viralCardImpressions,viralClicks,viralCommentLikes,viralComments,viralCompanyPageClicks,viralExternalWebsiteConversions,viralExternalWebsitePostClickConversions,viralExternalWebsitePostViewConversions,viralFollows,dateRange,pivot,pivotValue", + "dateRange": { + "start.day": 31, + "start.month": 8, + "start.year": 2021, + "end.day": 30, + "end.month": 9, + "end.year": 2021, + }, + }, + { + "camp_id": 123, + "fields": "viralFullScreenPlays,viralImpressions,viralLandingPageClicks,viralLikes,viralOneClickLeadFormOpens,viralOneClickLeads,viralOtherEngagements,viralReactions,viralShares,viralTotalEngagements,viralVideoCompletions,viralVideoFirstQuartileCompletions,viralVideoMidpointCompletions,viralVideoStarts,viralVideoThirdQuartileCompletions,viralVideoViews,dateRange,pivot,pivotValue", + "dateRange": { + "start.day": 1, + "start.month": 8, + "start.year": 2021, + "end.day": 31, + "end.month": 8, + "end.year": 2021, + }, + }, + { + "camp_id": 123, + "fields": "viralFullScreenPlays,viralImpressions,viralLandingPageClicks,viralLikes,viralOneClickLeadFormOpens,viralOneClickLeads,viralOtherEngagements,viralReactions,viralShares,viralTotalEngagements,viralVideoCompletions,viralVideoFirstQuartileCompletions,viralVideoMidpointCompletions,viralVideoStarts,viralVideoThirdQuartileCompletions,viralVideoViews,dateRange,pivot,pivotValue", + "dateRange": { + "start.day": 31, + "start.month": 8, + "start.year": 2021, + "end.day": 30, + "end.month": 9, + "end.year": 2021, + }, + }, +] + + +""" This is the example of the input chunks for the `test_merge_chunks` """ +test_input_result_record_chunks = [ + [ + { + "field_1": "test1", + "start_date": "2021-08-06", + "end_date": "2021-08-06", + }, + { + "field_1": "test2", + "start_date": "2021-08-07", + "end_date": "2021-08-07", + }, + { + "field_1": "test3", + "start_date": "2021-08-08", + "end_date": "2021-08-08", + }, + ], + [ + { + "field_2": "test1", + "start_date": "2021-08-06", + "end_date": "2021-08-06", + }, + { + "field_2": "test2", + "start_date": "2021-08-07", + "end_date": "2021-08-07", + }, + { + "field_2": "test3", + "start_date": "2021-08-08", + "end_date": "2021-08-08", + }, + ], + [ + { + "field_3": "test1", + "start_date": "2021-08-06", + "end_date": "2021-08-06", + }, + { + "field_3": "test2", + "start_date": "2021-08-07", + "end_date": "2021-08-07", + }, + { + "field_3": "test3", + "start_date": "2021-08-08", + "end_date": "2021-08-08", + }, + ], +] + +""" This is the expected test ouptput from the `merge_chunks` method from analytics module """ +test_output_merged_chunks = [ + { + "field_1": "test1", + "start_date": "2021-08-06", + "end_date": "2021-08-06", + "field_2": "test1", + "field_3": "test1", + }, + { + "field_1": "test2", + "start_date": "2021-08-07", + "end_date": "2021-08-07", + "field_2": "test2", + "field_3": "test2", + }, + { + "field_1": "test3", + "start_date": "2021-08-08", + "end_date": "2021-08-08", + "field_2": "test3", + "field_3": "test3", + }, +] diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_chunk_analytics_fields.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_chunk_analytics_fields.py new file mode 100644 index 000000000000..aa464dbf3d01 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_chunk_analytics_fields.py @@ -0,0 +1,59 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from source_linkedin_ads.analytics import chunk_analytics_fields + +# Test chunk size for each field set +TEST_FIELDS_CHUNK_SIZE = 3 +# Test fields assuming they are really available for the fetch +TEST_ANALYTICS_FIELDS = [ + "field_1", + "base_field_1", + "field_2", + "base_field_2", + "field_3", + "field_4", + "field_5", + "field_6", + "field_7", + "field_8", +] +# Fields that are always present in fields_set chunks +TEST_BASE_ANALLYTICS_FIELDS = ["base_field_1", "base_field_2"] + + +def test_chunk_analytics_fields(): + """ + We expect to truncate the fields list into the chunks of equal size, + with TEST_BASE_ANALLYTICS_FIELDS presence in each chunk, + order is not matter. + """ + expected_output = [ + ["field_1", "base_field_1", "field_2", "base_field_2"], + ["base_field_2", "field_3", "field_4", "base_field_1"], + ["field_5", "field_6", "field_7", "base_field_1", "base_field_2"], + ["field_8", "base_field_1", "base_field_2"], + ] + + assert list(chunk_analytics_fields(TEST_ANALYTICS_FIELDS, TEST_BASE_ANALLYTICS_FIELDS, TEST_FIELDS_CHUNK_SIZE)) == expected_output diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_make_analytics_slices.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_make_analytics_slices.py new file mode 100644 index 000000000000..02f93a535958 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_make_analytics_slices.py @@ -0,0 +1,38 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from samples.test_data_for_analytics import test_input_record, test_output_slices +from source_linkedin_ads.analytics import make_analytics_slices + +# Test input arguments for the `make_analytics_slices` +TEST_KEY_VALUE_MAP = {"camp_id": "id"} +TEST_START_DATE = "2021-08-01" +TEST_END_DATE = "2021-09-30" + +# This is the mock of the request_params +TEST_REQUEST_PRAMS = {} + + +def test_make_analytics_slices(): + assert list(make_analytics_slices(test_input_record, TEST_KEY_VALUE_MAP, TEST_START_DATE, TEST_END_DATE)) == test_output_slices diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_make_date_slices.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_make_date_slices.py new file mode 100644 index 000000000000..760e345acdab --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_make_date_slices.py @@ -0,0 +1,44 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from source_linkedin_ads.analytics import make_date_slices + +TEST_START_DATE = "2021-08-01" +TEST_END_DATE = "2021-10-01" + + +def test_make_date_slices(): + """ + : By default we use the `WINDOW_SIZE = 30`, as it set in the analytics module + : This value could be changed by setting the corresponding argument in the method. + : The `end_date` is not specified by default, but for this test it was specified to have the test static. + """ + + expected_output = [ + {"dateRange": {"start.day": 1, "start.month": 8, "start.year": 2021, "end.day": 31, "end.month": 8, "end.year": 2021}}, + {"dateRange": {"start.day": 31, "start.month": 8, "start.year": 2021, "end.day": 30, "end.month": 9, "end.year": 2021}}, + {"dateRange": {"start.day": 30, "start.month": 9, "start.year": 2021, "end.day": 30, "end.month": 10, "end.year": 2021}}, + ] + + assert list(make_date_slices(TEST_START_DATE, TEST_END_DATE)) == expected_output diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_merge_chunks.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_merge_chunks.py new file mode 100644 index 000000000000..a8648c747135 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_merge_chunks.py @@ -0,0 +1,33 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from samples.test_data_for_analytics import test_input_result_record_chunks, test_output_merged_chunks +from source_linkedin_ads.analytics import merge_chunks + +TEST_MERGE_BY_KEY = "end_date" + + +def test_merge_chunks(): + """ `merge_chunks` is the generator object, to get the output the list() function is applied """ + assert list(merge_chunks(test_input_result_record_chunks, TEST_MERGE_BY_KEY)) == test_output_merged_chunks diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py new file mode 100644 index 000000000000..a895a12af931 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py @@ -0,0 +1,135 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +""" This is the example of input record for the test_tranform_data. """ +input_test_data = [ + { + "targetingCriteria": { + "include": { + "and": [ + { + "or": { + "urn:li:adTargetingFacet:titles": [ + "urn:li:title:100", + "urn:li:title:10326", + "urn:li:title:10457", + "urn:li:title:10738", + "urn:li:title:10966", + "urn:li:title:11349", + "urn:li:title:1159", + ] + } + }, + {"or": {"urn:li:adTargetingFacet:locations": ["urn:li:geo:103644278"]}}, + {"or": {"urn:li:adTargetingFacet:interfaceLocales": ["urn:li:locale:en_US"]}}, + ] + }, + "exclude": { + "or": { + "urn:li:adTargetingFacet:facet_Key1": [ + "facet_test1", + "facet_test2", + ], + "urn:li:adTargetingFacet:facet_Key2": [ + "facet_test3", + "facet_test4", + ], + } + }, + }, + "changeAuditStamps": { + "created": {"time": 1629581275000}, + "lastModified": {"time": 1629664544760}, + }, + "dateRange": { + "start": {"month": 8, "day": 13, "year": 2021}, + "end": {"month": 8, "day": 13, "year": 2021}, + }, + "variables": { + "data": { + "com.linkedin.ads.SponsoredUpdateCreativeVariables": { + "activity": "urn:li:activity:1234", + "directSponsoredContent": 0, + "share": "urn:li:share:1234", + } + } + }, + } +] + +""" This is the expected output from the `transform_data` method. """ +output_test_data = [ + { + "targetingCriteria": { + "include": { + "and": [ + { + "type": "urn:li:adTargetingFacet:titles", + "values": [ + "urn:li:title:100", + "urn:li:title:10326", + "urn:li:title:10457", + "urn:li:title:10738", + "urn:li:title:10966", + "urn:li:title:11349", + "urn:li:title:1159", + ], + }, + { + "type": "urn:li:adTargetingFacet:locations", + "values": ["urn:li:geo:103644278"], + }, + { + "type": "urn:li:adTargetingFacet:interfaceLocales", + "values": ["urn:li:locale:en_US"], + }, + ] + }, + "exclude": { + "or": [ + { + "type": "urn:li:adTargetingFacet:facet_Key1", + "values": ["facet_test1", "facet_test2"], + }, + { + "type": "urn:li:adTargetingFacet:facet_Key2", + "values": ["facet_test3", "facet_test4"], + }, + ] + }, + }, + "variables": { + "type": "com.linkedin.ads.SponsoredUpdateCreativeVariables", + "values": [ + {"key": "activity", "value": "urn:li:activity:1234"}, + {"key": "directSponsoredContent", "value": 0}, + {"key": "share", "value": "urn:li:share:1234"}, + ], + }, + "created": "2021-08-21 21:27:55", + "lastModified": "2021-08-22 20:35:44", + "start_date": "2021-08-13", + "end_date": "2021-08-13", + } +] diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/test_make_slice.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/test_make_slice.py new file mode 100644 index 000000000000..4bc55c5fac32 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/test_make_slice.py @@ -0,0 +1,37 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +import pytest +from source_linkedin_ads.utils import get_parent_stream_values + + +@pytest.mark.parametrize( + "record, key_value_map, output_slice", + [ + ({"id": 123, "ref": "abc"}, {"acc_id": "id"}, {"acc_id": 123}), + ({"id": 123, "ref": "abc"}, {"acc_id": "id", "ref_id": "ref"}, {"acc_id": 123, "ref_id": "abc"}), + ], +) +def test_get_parent_stream_values(record, key_value_map, output_slice): + assert get_parent_stream_values(record, key_value_map) == output_slice diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/test_transform_data.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/test_transform_data.py new file mode 100644 index 000000000000..f8eb1c1f5016 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/test_transform_data.py @@ -0,0 +1,34 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from samples.test_data_for_tranform import input_test_data, output_test_data +from source_linkedin_ads.utils import transform_data + + +def test_transfrom_data(): + """ + As far as we transform the data within the generator object, + we use list() to have the actual output for the test assertion. + """ + assert list(transform_data(input_test_data)) == output_test_data diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 47cb33cf795f..dda568c341b5 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -74,6 +74,7 @@ * [Kafka](integrations/sources/kafka.md) * [Klaviyo](integrations/sources/klaviyo.md) * [Kustomer](integrations/sources/kustomer.md) + * [LinkedIn Ads](integrations/sources/linkedin-ads.md) * [Lever Hiring](integrations/sources/lever-hiring.md) * [Looker](integrations/sources/looker.md) * [Magento](integrations/sources/magento.md) diff --git a/docs/integrations/README.md b/docs/integrations/README.md index c1f40b4e77f6..6f57370faf04 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -54,6 +54,8 @@ Airbyte uses a grading system for connectors to help users understand what to ex |[Iterable](./sources/iterable.md)| Beta | |[Jira](./sources/jira.md)| Certified | |[Klaviyo](./sources/klaviyo.md)| Beta | +|[Klaviyo](./sources/kustomer.md)| Alpha | +|[LinkedIn Ads](./sources/linkedin-ads.md)| Beta | |[Kustomer](./sources/kustomer.md)| Alpha | |[Lever Hiring](./sources/lever-hiring.md)| Beta | |[Looker](./sources/looker.md)| Beta | diff --git a/docs/integrations/sources/linkedin-ads.md b/docs/integrations/sources/linkedin-ads.md new file mode 100644 index 000000000000..c3917658c586 --- /dev/null +++ b/docs/integrations/sources/linkedin-ads.md @@ -0,0 +1,135 @@ +# LinkedIn Ads + +## Sync overview + +The LinkedIn Ads source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. + +This Source Connector is based on a [Airbyte CDK](https://docs.airbyte.io/connector-development/cdk-python). +Airbyte uses [LinkedIn Marketing Developer Platform - API](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/marketing-integrations-overview) to fetch data from LinkedIn Ads. + +### Output schema + +This Source is capable of syncing the following data as streams: +* [Accounts](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads/account-structure/create-and-manage-accounts?tabs=http#search-for-accounts) +* [Account Users](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads/account-structure/create-and-manage-account-users?tabs=http#find-ad-account-users-by-accounts) +* [Campaign Groups](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads/account-structure/create-and-manage-campaign-groups?tabs=http#search-for-campaign-groups) +* [Campaigns](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads/account-structure/create-and-manage-campaigns?tabs=http#search-for-campaigns) +* [Creatives](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads/account-structure/create-and-manage-creatives?tabs=http#search-for-creatives) +* [Ad Direct Sponsored Contents](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads/advertising-targeting/create-and-manage-video?tabs=http#finders) +* [Ad Analytics by Campaign](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?tabs=curl#ad-analytics) +* [Ad Analytics by Creative](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?tabs=curl#ad-analytics) + + +### NOTE: +`Ad Direct Sponsored Contents` includes the information about VIDEO ADS, as well as SINGLE IMAGE ADS and other directly sponsored ads your account might have. + +### Data type mapping + +| Integration Type | Airbyte Type | Notes | +| :--- | :--- | :--- | +| `number` | `number` | float number | +| `integer` | `integer` | whole number | +| `date` | `string` | FORMAT YYYY-MM-DD | +| `datetime` | `string` | FORMAT YYYY-MM-DDThh:mm:ss | +| `array` | `array` | | +| `boolean` | `boolean` | True/False | +| `string` | `string` | | + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :--- | :--- | :--- | +| Full Refresh Overwrite Sync | Yes | | +| Full Refresh Append Sync | Yes | | +| Incremental - Append Sync | Yes | | +| Incremental - Append + Deduplication Sync | Yes | | +| Namespaces | No | | + + +### Performance considerations + +There are official Rate Limits for LinkedIn Ads API Usage, [more information here](https://docs.microsoft.com/en-us/linkedin/shared/api-guide/concepts/rate-limits?context=linkedin/marketing/context). +Rate limited requests will receive a 429 response. In rare cases, LinkedIn may also return a 429 response as part of infrastructure protection. API service will return to normal automatically. +In such cases you will receive the next error message: +``` +"Caught retryable error ' or null' after tries. Waiting seconds then retrying..." +``` +This is expected when the connector hits the 429 - Rate Limit Exceeded HTTP Error. +If the maximum of available API requests capacity is reached, you will have the following message: +``` +"Max try rate limit exceded..." +``` +After 5 unsuccessful attempts - the connector will stop the sync operation. +In such cases check your Rate Limits [on this page](https://www.linkedin.com/developers/apps) > Choose you app > Analytics + + +## Getting started + +### Authentication +The source LinkedIn uses `access_token` provided in the UI connector's settings to make API requests. Access tokens expire after `2 months from generating date (60 days)` and require a user to manually authenticate again. If you receive a `401 invalid token response`, the error logs will state that your access token has expired and to re-authenticate your connection to generate a new token. This is described more [here](https://docs.microsoft.com/en-us/linkedin/shared/authentication/authorization-code-flow?context=linkedin/context). + +The API user account should be assigned one of the following roles: +* ACCOUNT_BILLING_ADMIN +* ACCOUNT_MANAGER +* CAMPAIGN_MANAGER +* CREATIVE_MANAGER +* VIEWER (Recommended) + +The API user account should be assigned the following permissions for the API endpoints: + +Endpoints such as: +`Accounts`, `Account Users`, `Ad Direct Sponsored Contents`, `Campaign Groups`, `Campaings`, `Creatives` requires the next permissions set: + + * `r_ads`: read ads (Recommended), `rw_ads`: read-write ads + +Endpoints such as: `Ad Analytics by Campaign`, `Ad Analytics by Creatives` requires the next permissions set: + + * `r_ads_reporting`: read ads reporting + +The complete set of prmissions is: + * `r_emailaddress,r_liteprofile,r_ads,r_ads_reporting,r_organization_social` + +### Generate the Access_Token +1. **Login to LinkedIn as the API user.** +2. **Create an App** [here](https://www.linkedin.com/developers/apps): + * `App Name`: airbyte-source + * `Company`: search and find your company LinkedIn page + * `Privacy policy URL`: link to company privacy policy + * `Business email`: developer/admin email address + * `App logo`: Airbyte's (or Company's) logo + * `Products`: Select [Marketing Developer Platform](https://www.linkedin.com/developers/apps/122632736/products/marketing-developer-platform) (checkbox) + Review/agree to legal terms and create app. +3. **Verify App**: + * Provide the verify URL to your Company's LinkedIn Admin to verify and authorize the app. + * Once verified, select the App in the Console [here](https://www.linkedin.com/developers/apps). + * **Review the `Auth` tab**: + * Record `client_id` and `client_secret` (for later steps). + * Review permissions and ensure app has the permissions (above). + * Oauth 2.0 settings: Provide a `redirect_uri` (for later steps): `https://airbyte.io` + * Review the `Products` tab and ensure `Marketing Developer Platform` has been added and approved (listed in the `Products` section/tab). + * Review the `Usage & limits` tab. This shows the daily application and user/member limits with percent used for each resource endpoint. +4. **Authorize App**: (The authorization token `lasts 60-days before expiring`. The connector app will need to be reauthorized when the authorization token expires.): + Create an Authorization URL with the following pattern: + * The permissions set you need to use is: `r_emailaddress,r_liteprofile,r_ads,r_ads_reporting,r_organization_social` + * URL pattern: Provide the scope from permissions above (with + delimiting each permission) and replace the other highlighted parameters: `https://www.linkedin.com/oauth/v2/authorization?response_type=code&client_id=YOUR_CLIENT_ID&redirect_uri=YOUR_REDIRECT_URI&scope=r_emailaddress,r_liteprofile,r_ads,r_ads_reporting,r_organization_social` + * Modify and open the `url` in the browser. + * Once redirected, click `Allow` to authorize app. + * The browser will be redirected to the `redirect_uri`. Record the `code` parameter listed in the redirect URL in the Browser header URL. +5. **Run the following curl command** using `Terminal` or `Command line` with the parameters replaced to return your `access_token`. The `access_token` expires in 2-months. + ``` + curl -0 -v -X POST https://www.linkedin.com/oauth/v2/accessToken\ + -H "Accept: application/json"\ + -H "application/x-www-form-urlencoded"\ + -d "grant_type=authorization_code"\ + -d "code=YOUR_CODE"\ + -d "client_id=YOUR_CLIENT_ID"\ + -d "client_secret=YOUR_CLIENT_SECRET"\ + -d "redirect_uri=YOUR_REDIRECT_URI" + ``` +6. **Use the `access_token`** from response from the `Step 5` to autorize LinkedIn Ads connector. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :-------- | :-------- | :------ | +| 0.1.0 | 2021-09-05 | [5285](https://github.com/airbytehq/airbyte/pull/5285) | Initial release of Native LinkedIn Ads connector for Airbyte | diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index 8125aacfa82d..324271012f20 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -89,6 +89,7 @@ write_standard_creds source-jira "$JIRA_INTEGRATION_TEST_CREDS" write_standard_creds source-klaviyo "$KLAVIYO_TEST_CREDS" write_standard_creds source-lever-hiring "$LEVER_HIRING_INTEGRATION_TEST_CREDS" write_standard_creds source-looker "$LOOKER_INTEGRATION_TEST_CREDS" +write_standard_creds source-linkedin-ads "$SOURCE_LINKEDIN_ADS_TEST_CREDS" write_standard_creds source-mailchimp "$MAILCHIMP_TEST_CREDS" write_standard_creds source-marketo "$SOURCE_MARKETO_TEST_CREDS" write_standard_creds source-marketo-singer "$SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG"