From c46d2e6b4516e357c39f1baa6bb97b530ebc0d86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xabier=20Lahuerta=20V=C3=A1zquez?= Date: Thu, 27 Oct 2022 23:32:02 +0200 Subject: [PATCH 1/8] Initial Newsdata source connector implementation --- .../connectors/source-newsdata/.dockerignore | 6 + .../connectors/source-newsdata/Dockerfile | 38 ++++ .../connectors/source-newsdata/README.md | 79 +++++++ .../connectors/source-newsdata/__init__.py | 3 + .../acceptance-test-config.yml | 27 +++ .../source-newsdata/acceptance-test-docker.sh | 16 ++ .../connectors/source-newsdata/build.gradle | 9 + .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 5 + .../integration_tests/acceptance.py | 14 ++ .../integration_tests/configured_catalog.json | 22 ++ .../integration_tests/invalid_config.json | 4 + .../integration_tests/sample_config.json | 4 + .../integration_tests/sample_state.json | 5 + .../connectors/source-newsdata/main.py | 13 ++ .../source-newsdata/requirements.txt | 2 + .../connectors/source-newsdata/setup.py | 29 +++ .../source_newsdata/__init__.py | 8 + .../source_newsdata/newsdata.yaml | 66 ++++++ .../source_newsdata/schemas/latest.json | 57 ++++++ .../source_newsdata/schemas/sources.json | 33 +++ .../source-newsdata/source_newsdata/source.py | 18 ++ .../source-newsdata/source_newsdata/spec.yaml | 192 ++++++++++++++++++ docs/integrations/README.md | 1 + docs/integrations/sources/newsdata.md | 46 +++++ 25 files changed, 700 insertions(+) create mode 100644 airbyte-integrations/connectors/source-newsdata/.dockerignore create mode 100644 airbyte-integrations/connectors/source-newsdata/Dockerfile create mode 100644 airbyte-integrations/connectors/source-newsdata/README.md create mode 100644 airbyte-integrations/connectors/source-newsdata/__init__.py create mode 100644 airbyte-integrations/connectors/source-newsdata/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-newsdata/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-newsdata/build.gradle create mode 100644 airbyte-integrations/connectors/source-newsdata/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-newsdata/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-newsdata/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-newsdata/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-newsdata/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-newsdata/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-newsdata/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-newsdata/main.py create mode 100644 airbyte-integrations/connectors/source-newsdata/requirements.txt create mode 100644 airbyte-integrations/connectors/source-newsdata/setup.py create mode 100644 airbyte-integrations/connectors/source-newsdata/source_newsdata/__init__.py create mode 100644 airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml create mode 100644 airbyte-integrations/connectors/source-newsdata/source_newsdata/schemas/latest.json create mode 100644 airbyte-integrations/connectors/source-newsdata/source_newsdata/schemas/sources.json create mode 100644 airbyte-integrations/connectors/source-newsdata/source_newsdata/source.py create mode 100644 airbyte-integrations/connectors/source-newsdata/source_newsdata/spec.yaml create mode 100644 docs/integrations/sources/newsdata.md diff --git a/airbyte-integrations/connectors/source-newsdata/.dockerignore b/airbyte-integrations/connectors/source-newsdata/.dockerignore new file mode 100644 index 000000000000..e53943a36e18 --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_newsdata +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-newsdata/Dockerfile b/airbyte-integrations/connectors/source-newsdata/Dockerfile new file mode 100644 index 000000000000..5fe81e4686ed --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_newsdata ./source_newsdata + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-newsdata diff --git a/airbyte-integrations/connectors/source-newsdata/README.md b/airbyte-integrations/connectors/source-newsdata/README.md new file mode 100644 index 000000000000..4a4e36cc6c3a --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/README.md @@ -0,0 +1,79 @@ +# Newsdata Source + +This is the repository for the Newsdata configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/newsdata). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-newsdata:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/newsdata) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_newsdata/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source newsdata test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-newsdata:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-newsdata:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-newsdata:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-newsdata:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-newsdata:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-newsdata:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-newsdata:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-newsdata:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-newsdata/__init__.py b/airbyte-integrations/connectors/source-newsdata/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-newsdata/acceptance-test-config.yml b/airbyte-integrations/connectors/source-newsdata/acceptance-test-config.yml new file mode 100644 index 000000000000..a333228b391b --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/acceptance-test-config.yml @@ -0,0 +1,27 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-newsdata:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_newsdata/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + incremental: + bypass_reason: "This connector does not implement incremental sync" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-newsdata/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-newsdata/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-newsdata/build.gradle b/airbyte-integrations/connectors/source-newsdata/build.gradle new file mode 100644 index 000000000000..8e930b32624b --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_newsdata' +} diff --git a/airbyte-integrations/connectors/source-newsdata/integration_tests/__init__.py b/airbyte-integrations/connectors/source-newsdata/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-newsdata/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-newsdata/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-newsdata/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-newsdata/integration_tests/acceptance.py new file mode 100644 index 000000000000..950b53b59d41 --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-newsdata/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-newsdata/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..bf69b4357da5 --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/integration_tests/configured_catalog.json @@ -0,0 +1,22 @@ +{ + "streams": [ + { + "stream": { + "name": "latest", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "sources", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-newsdata/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-newsdata/integration_tests/invalid_config.json new file mode 100644 index 000000000000..9266d77f33fd --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "ivalid_api_key", + "category": ["invalid_category_1", "invalid_category_2"] +} diff --git a/airbyte-integrations/connectors/source-newsdata/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-newsdata/integration_tests/sample_config.json new file mode 100644 index 000000000000..a216b34cceaa --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/integration_tests/sample_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "TODO: set your api_key", + "query": "pizza" +} diff --git a/airbyte-integrations/connectors/source-newsdata/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-newsdata/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-newsdata/main.py b/airbyte-integrations/connectors/source-newsdata/main.py new file mode 100644 index 000000000000..6141f19bc0f4 --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_newsdata import SourceNewsdata + +if __name__ == "__main__": + source = SourceNewsdata() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-newsdata/requirements.txt b/airbyte-integrations/connectors/source-newsdata/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-newsdata/setup.py b/airbyte-integrations/connectors/source-newsdata/setup.py new file mode 100644 index 000000000000..e37e1b508e46 --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_newsdata", + description="Source implementation for Newsdata.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-newsdata/source_newsdata/__init__.py b/airbyte-integrations/connectors/source-newsdata/source_newsdata/__init__.py new file mode 100644 index 000000000000..117cedb9b0e8 --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/source_newsdata/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceNewsdata + +__all__ = ["SourceNewsdata"] diff --git a/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml b/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml new file mode 100644 index 000000000000..e1a93bc403ed --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml @@ -0,0 +1,66 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: ["results"] + base_requester: + url_base: "https://newsdata.io/api/1" + http_method: "GET" + authenticator: + type: ApiKeyAuthenticator + header: "X-ACCESS-KEY" + api_token: "{{ config['api_key'] }}" + base_retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: NoPagination + base_stream: + retriever: + $ref: "*ref(definitions.base_retriever)" + requester: + $ref: "*ref(definitions.base_requester)" + common_inputs: + country: "{{ ','.join(config['country']) }}" + language: "{{ ','.join(config['language']) }}" + category: "{{ ','.join(config['category']) }}" + latest_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "latest" + primary_key: "link" + path: "/news" + retriever: + $ref: "*ref(definitions.base_retriever)" + requester: + $ref: "*ref(definitions.base_requester)" + request_options_provider: + request_parameters: + $ref: "*ref(definitions.common_inputs)" + q: "{{ config['query'] | urlencode }}" + qInTitle: "{{ config['queryInTitle'] | urlencode }}" + domain: "{{ ','.join(config['domain']) }}" + sources_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "sources" + primary_key: "id" + path: "/sources" + retriever: + $ref: "*ref(definitions.base_retriever)" + requester: + $ref: "*ref(definitions.base_requester)" + request_options_provider: + request_parameters: + $ref: "*ref(definitions.common_inputs)" + + +streams: + - "*ref(definitions.latest_stream)" + - "*ref(definitions.sources_stream)" + +check: + stream_names: + - "latest" + - "sources" diff --git a/airbyte-integrations/connectors/source-newsdata/source_newsdata/schemas/latest.json b/airbyte-integrations/connectors/source-newsdata/source_newsdata/schemas/latest.json new file mode 100644 index 000000000000..42aa64c9db68 --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/source_newsdata/schemas/latest.json @@ -0,0 +1,57 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "title": { + "type": ["null", "string"] + }, + "link": { + "type": ["null", "string"] + }, + "source_id": { + "type": ["null", "string"] + }, + "keywords": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "creator": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "image_url": { + "type": ["null", "string"] + }, + "video_url": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "pubDate": { + "type": ["null", "string"] + }, + "content": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "category": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "language": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-newsdata/source_newsdata/schemas/sources.json b/airbyte-integrations/connectors/source-newsdata/source_newsdata/schemas/sources.json new file mode 100644 index 000000000000..d50841fc0ef1 --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/source_newsdata/schemas/sources.json @@ -0,0 +1,33 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "category": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "language": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "country": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-newsdata/source_newsdata/source.py b/airbyte-integrations/connectors/source-newsdata/source_newsdata/source.py new file mode 100644 index 000000000000..bba2fc179366 --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/source_newsdata/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceNewsdata(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "newsdata.yaml"}) diff --git a/airbyte-integrations/connectors/source-newsdata/source_newsdata/spec.yaml b/airbyte-integrations/connectors/source-newsdata/source_newsdata/spec.yaml new file mode 100644 index 000000000000..8987ee36a188 --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/source_newsdata/spec.yaml @@ -0,0 +1,192 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/newsdata +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Newsdata Spec + type: object + required: + - api_key + additionalProperties: true + properties: + api_key: + type: string + description: API Key + airbyte_secret: true + order: 0 + query: + type: string + description: >- + Keywords or phrases to search for in the news title and content. + Advanced Search options: + - Search `Social`: query = "social" + - Search `Social Pizza`: query = "social pizza" + - Search `Social` but not with `pizza`: query = "social -pizza" + - Search `Social` but not with `pizza` and `wildfire`: query = "social -pizza -wildfire" + - Search `Social` and `pizza`: query = "social AND pizza" + - Search `Social` and `pizza` and `pasta`: query = "social AND pizza AND pasta" + - Search `Social` or `pizza`: query = "social OR pizza" + - Search `Social` or `pizza` but not `pasta`: query = "social OR pizza -pasta" + - Search `Social` or `pizza` or `pasta`: query = "social OR pizza OR pasta" + Note: You can't use AND and OR in the same query. + order: 1 + queryInTitle: + type: string + description: Same as `query`, but restricting the search to only the news title. It cannot be used along with `query`. + domain: + type: array + description: Domains (maximum 5) to restrict the search to. Use the sources stream to find top sources id. + items: + type: string + order: 3 + country: + type: array + description: 2-letter ISO 3166-1 countries (maximum 5) to restrict the search to. + order: 4 + items: + type: string + enum: + - ar + - au + - at + - bd + - by + - be + - br + - bg + - ca + - cl + - cn + - co + - cr + - cu + - cz + - dk + - do + - ec + - eg + - ee + - et + - fi + - fr + - de + - gr + - hk + - hu + - in + - id + - iq + - ie + - il + - it + - jp + - kz + - kw + - lv + - lb + - lt + - my + - mx + - ma + - mm + - nl + - nz + - ng + - kp + - no + - pk + - pe + - ph + - pl + - pt + - pr + - ro + - ru + - sa + - rs + - sg + - sk + - si + - za + - kr + - es + - se + - ch + - tw + - tz + - th + - tr + - ua + - ae + - gb + - us + - ve + - vi + category: + type: array + description: Categories (maximum 5) to restrict the search to. + order: 5 + items: + type: string + enum: + - business + - entertainment + - environment + - food + - health + - politics + - science + - sports + - technology + - top + - world + language: + type: array + description: Languages (maximum 5) to restrict the search to. + order: 6 + items: + type: string + enum: + - be + - am + - ar + - bn + - bs + - bg + - my + - ckb + - zh + - hr + - cs + - da + - nl + - en + - et + - fi + - fr + - de + - el + - he + - hi + - hu + - in + - it + - jp + - ko + - lv + - lt + - ms + - no + - pl + - pt + - ro + - ru + - sr + - sk + - sl + - es + - sw + - sv + - th + - tr + - uk + - ur + - vi \ No newline at end of file diff --git a/docs/integrations/README.md b/docs/integrations/README.md index 26454b792a87..e7a0820ef6d8 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -112,6 +112,7 @@ For more information about the grading system, see [Product Release Stages](http | [Mongo DB](sources/mongodb-v2.md) | Alpha | Yes | | [My Hours](sources/my-hours.md) | Alpha | Yes | | [MySQL](sources/mysql.md) | Beta | Yes | +| [Newsdata](sources/newsdata.md) | Alpha | Yes | | [Notion](sources/notion.md) | Generally Available | Yes | | [Okta](sources/okta.md) | Alpha | Yes | | [OneSignal](sources/onesignal.md) | Alpha | No | diff --git a/docs/integrations/sources/newsdata.md b/docs/integrations/sources/newsdata.md new file mode 100644 index 000000000000..20e764d2f759 --- /dev/null +++ b/docs/integrations/sources/newsdata.md @@ -0,0 +1,46 @@ +# Newsdata API + +## Sync overview + +This source retrieves the latests news from the [Newsdata API](https://newsdata.io/). + +### Output schema + +This source is capable of syncing the following streams: + +* `latest` +* `sources` + +If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose) + +### Features + +| Feature | Supported? | Notes | +|:------------------|------------|:------| +| Full Refresh Sync | Yes | | +| Incremental Sync | No | | + +### Performance considerations + +The News API free tier only allows 200 requests per day, and only up to 10 +news per request. + +The free tier does not allow to perform advanced search queries. + +## Getting started + +### Requirements + +1. A Newsdata API key. You can get one [here](https://newsdata.io/register). + +### Setup guide + +The following fields are required fields for the connector to work: + +- `api_key`: Your Newsdata API key. + +## Changelog +TODO +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-----------| +| 0.1.0 | 2022-10-21 | [](https://github.com/airbytehq/airbyte/pull/) | 🎉 New Source: Newsdata | From 802a0ce67dd1ce454f2b9b5dcd4d6885a62ab7f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xabier=20Lahuerta=20V=C3=A1zquez?= Date: Fri, 28 Oct 2022 01:23:34 +0200 Subject: [PATCH 2/8] Update docs, source_definitions and upload icon --- .../src/main/resources/icons/newsdata.svg | 89 +++++++++++++++++++ .../resources/seed/source_definitions.yaml | 7 ++ docs/integrations/sources/newsdata.md | 8 +- 3 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 airbyte-config/init/src/main/resources/icons/newsdata.svg diff --git a/airbyte-config/init/src/main/resources/icons/newsdata.svg b/airbyte-config/init/src/main/resources/icons/newsdata.svg new file mode 100644 index 000000000000..2afed87c1fd0 --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/newsdata.svg @@ -0,0 +1,89 @@ + + + + + \ No newline at end of file diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 5e137848006f..10c2e74816be 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -754,6 +754,13 @@ documentationUrl: https://docs.airbyte.com/integrations/sources/news-api sourceType: api releaseStage: alpha +- name: Newsdata + sourceDefinitionId: 60bd11d8-2632-4daa-a688-b47336d32093 + dockerRepository: airbyte/source-newsdata + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/newsdata + sourceType: api + releaseStage: alpha - name: Notion sourceDefinitionId: 6e00b415-b02e-4160-bf02-58176a0ae687 dockerRepository: airbyte/source-notion diff --git a/docs/integrations/sources/newsdata.md b/docs/integrations/sources/newsdata.md index 20e764d2f759..023e31ac501f 100644 --- a/docs/integrations/sources/newsdata.md +++ b/docs/integrations/sources/newsdata.md @@ -40,7 +40,7 @@ The following fields are required fields for the connector to work: - `api_key`: Your Newsdata API key. ## Changelog -TODO -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:-----------| -| 0.1.0 | 2022-10-21 | [](https://github.com/airbytehq/airbyte/pull/) | 🎉 New Source: Newsdata | + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:------------------------| +| 0.1.0 | 2022-10-21 | [18576](https://github.com/airbytehq/airbyte/pull/18576) | 🎉 New Source: Newsdata | From 8f29c2fecf86cfb09082591478f8aaba61234fcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xabier=20Lahuerta=20V=C3=A1zquez?= Date: Fri, 28 Oct 2022 12:42:45 +0200 Subject: [PATCH 3/8] Set single input for `sources` stream --- .../source-newsdata/source_newsdata/newsdata.yaml | 11 ++++++----- docs/integrations/sources/newsdata.md | 4 +++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml b/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml index e1a93bc403ed..168f52f69902 100644 --- a/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml +++ b/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml @@ -22,9 +22,6 @@ definitions: requester: $ref: "*ref(definitions.base_requester)" common_inputs: - country: "{{ ','.join(config['country']) }}" - language: "{{ ','.join(config['language']) }}" - category: "{{ ','.join(config['category']) }}" latest_stream: $ref: "*ref(definitions.base_stream)" $options: @@ -37,7 +34,9 @@ definitions: $ref: "*ref(definitions.base_requester)" request_options_provider: request_parameters: - $ref: "*ref(definitions.common_inputs)" + country: "{{ ','.join(config['country']) }}" + language: "{{ ','.join(config['language']) }}" + category: "{{ ','.join(config['category']) }}" q: "{{ config['query'] | urlencode }}" qInTitle: "{{ config['queryInTitle'] | urlencode }}" domain: "{{ ','.join(config['domain']) }}" @@ -53,7 +52,9 @@ definitions: $ref: "*ref(definitions.base_requester)" request_options_provider: request_parameters: - $ref: "*ref(definitions.common_inputs)" + country: "{{ config['country'][0] }}" + language: "{{ config['language'][0] }}" + category: "{{ config['category'][0] }}" streams: diff --git a/docs/integrations/sources/newsdata.md b/docs/integrations/sources/newsdata.md index 023e31ac501f..959f6b5c8c40 100644 --- a/docs/integrations/sources/newsdata.md +++ b/docs/integrations/sources/newsdata.md @@ -9,7 +9,9 @@ This source retrieves the latests news from the [Newsdata API](https://newsdata. This source is capable of syncing the following streams: * `latest` -* `sources` +* `sources` + - __NOTE__: `category`, `language` and `country` input parameters only accept a single value, not multiple like `latest` stream. + Thus, if several values are supplied, the first one will be the one to be used. If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose) From 9310d49165972c8592f9e04d97960b9ad673c808 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xabier=20Lahuerta=20V=C3=A1zquez?= Date: Fri, 28 Oct 2022 12:44:07 +0200 Subject: [PATCH 4/8] Remove unused definition --- .../connectors/source-newsdata/source_newsdata/newsdata.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml b/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml index 168f52f69902..70144657c56b 100644 --- a/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml +++ b/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml @@ -21,7 +21,6 @@ definitions: $ref: "*ref(definitions.base_retriever)" requester: $ref: "*ref(definitions.base_requester)" - common_inputs: latest_stream: $ref: "*ref(definitions.base_stream)" $options: From ffc999caad2165488dc67ad6fe3d3cec563fb05b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xabier=20Lahuerta=20V=C3=A1zquez?= Date: Fri, 28 Oct 2022 12:54:57 +0200 Subject: [PATCH 5/8] Update `spec.yaml` config --- .../source_newsdata/newsdata.yaml | 2 +- .../source-newsdata/source_newsdata/spec.yaml | 54 ++++++++++--------- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml b/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml index 70144657c56b..02be4dd0aaf5 100644 --- a/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml +++ b/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml @@ -37,7 +37,7 @@ definitions: language: "{{ ','.join(config['language']) }}" category: "{{ ','.join(config['category']) }}" q: "{{ config['query'] | urlencode }}" - qInTitle: "{{ config['queryInTitle'] | urlencode }}" + qInTitle: "{{ config['query_in_title'] | urlencode }}" domain: "{{ ','.join(config['domain']) }}" sources_stream: $ref: "*ref(definitions.base_stream)" diff --git a/airbyte-integrations/connectors/source-newsdata/source_newsdata/spec.yaml b/airbyte-integrations/connectors/source-newsdata/source_newsdata/spec.yaml index 8987ee36a188..14d613d72c2c 100644 --- a/airbyte-integrations/connectors/source-newsdata/source_newsdata/spec.yaml +++ b/airbyte-integrations/connectors/source-newsdata/source_newsdata/spec.yaml @@ -12,35 +12,39 @@ connectionSpecification: description: API Key airbyte_secret: true order: 0 - query: - type: string - description: >- - Keywords or phrases to search for in the news title and content. - Advanced Search options: - - Search `Social`: query = "social" - - Search `Social Pizza`: query = "social pizza" - - Search `Social` but not with `pizza`: query = "social -pizza" - - Search `Social` but not with `pizza` and `wildfire`: query = "social -pizza -wildfire" - - Search `Social` and `pizza`: query = "social AND pizza" - - Search `Social` and `pizza` and `pasta`: query = "social AND pizza AND pasta" - - Search `Social` or `pizza`: query = "social OR pizza" - - Search `Social` or `pizza` but not `pasta`: query = "social OR pizza -pasta" - - Search `Social` or `pizza` or `pasta`: query = "social OR pizza OR pasta" - Note: You can't use AND and OR in the same query. - order: 1 - queryInTitle: - type: string - description: Same as `query`, but restricting the search to only the news title. It cannot be used along with `query`. + OneOf: + query: + type: string + description: >- + Keywords or phrases to search for in the news title and content. + Advanced Search options: + - Search `Social`: query = "social" + - Search `Social Pizza`: query = "social pizza" + - Search `Social` but not with `pizza`: query = "social -pizza" + - Search `Social` but not with `pizza` and `wildfire`: query = "social -pizza -wildfire" + - Search `Social` and `pizza`: query = "social AND pizza" + - Search `Social` and `pizza` and `pasta`: query = "social AND pizza AND pasta" + - Search `Social` or `pizza`: query = "social OR pizza" + - Search `Social` or `pizza` but not `pasta`: query = "social OR pizza -pasta" + - Search `Social` or `pizza` or `pasta`: query = "social OR pizza OR pasta" + Note: You can't use AND and OR in the same query. + order: 1 + query_in_title: + type: string + description: Same as `query`, but restricting the search to only the news title. It cannot be used along with `query`. + order: 1 domain: type: array description: Domains (maximum 5) to restrict the search to. Use the sources stream to find top sources id. + maxitems: 5 items: type: string - order: 3 + order: 2 country: type: array description: 2-letter ISO 3166-1 countries (maximum 5) to restrict the search to. - order: 4 + maxitems: 5 + order: 3 items: type: string enum: @@ -123,7 +127,8 @@ connectionSpecification: category: type: array description: Categories (maximum 5) to restrict the search to. - order: 5 + maxitems: 5 + order: 4 items: type: string enum: @@ -141,7 +146,8 @@ connectionSpecification: language: type: array description: Languages (maximum 5) to restrict the search to. - order: 6 + maxitems: 5 + order: 5 items: type: string enum: @@ -189,4 +195,4 @@ connectionSpecification: - tr - uk - ur - - vi \ No newline at end of file + - vi From eb19152a11a4f357a4c7d15fc45c8b5aefac3a94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xabier=20Lahuerta=20V=C3=A1zquez?= Date: Fri, 28 Oct 2022 17:37:59 +0200 Subject: [PATCH 6/8] Support Pagination --- .../source_newsdata/newsdata.yaml | 18 ++++++++++++++++-- .../source-newsdata/source_newsdata/spec.yaml | 18 ++++++++++++------ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml b/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml index 02be4dd0aaf5..1c897497fe6f 100644 --- a/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml +++ b/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml @@ -14,13 +14,25 @@ definitions: base_retriever: record_selector: $ref: "*ref(definitions.selector)" - paginator: - type: NoPagination base_stream: retriever: $ref: "*ref(definitions.base_retriever)" requester: $ref: "*ref(definitions.base_requester)" + cursor_paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response['nextPage'] }}" + page_size: 10 + # page_size: !!int "{% if config['is_free_tier'] %}{{ 10 }}{% else %}{{ 50 }}{% endif %}" + page_token_option: + field_name: "page" + inject_into: "request_parameter" + page_size_option: # This is useless, only there because it is required, but page sizes are managed automatically by API subscription type + field_name: "X-Pagination-Page-Size" + inject_into: "header" + url_base: "*ref(definitions.base_requester.url_base)" latest_stream: $ref: "*ref(definitions.base_stream)" $options: @@ -39,6 +51,8 @@ definitions: q: "{{ config['query'] | urlencode }}" qInTitle: "{{ config['query_in_title'] | urlencode }}" domain: "{{ ','.join(config['domain']) }}" + paginator: + $ref: "*ref(definitions.cursor_paginator)" sources_stream: $ref: "*ref(definitions.base_stream)" $options: diff --git a/airbyte-integrations/connectors/source-newsdata/source_newsdata/spec.yaml b/airbyte-integrations/connectors/source-newsdata/source_newsdata/spec.yaml index 14d613d72c2c..e7cb91011dd9 100644 --- a/airbyte-integrations/connectors/source-newsdata/source_newsdata/spec.yaml +++ b/airbyte-integrations/connectors/source-newsdata/source_newsdata/spec.yaml @@ -12,6 +12,12 @@ connectionSpecification: description: API Key airbyte_secret: true order: 0 + is_free_tier: + type: boolean + description: Indicates if API Key is free tier. This value affects page size, 10 if free tier, 50 if not. + default: true + airbyte_secret: true + order: 1 OneOf: query: type: string @@ -28,23 +34,23 @@ connectionSpecification: - Search `Social` or `pizza` but not `pasta`: query = "social OR pizza -pasta" - Search `Social` or `pizza` or `pasta`: query = "social OR pizza OR pasta" Note: You can't use AND and OR in the same query. - order: 1 + order: 2 query_in_title: type: string description: Same as `query`, but restricting the search to only the news title. It cannot be used along with `query`. - order: 1 + order: 2 domain: type: array description: Domains (maximum 5) to restrict the search to. Use the sources stream to find top sources id. maxitems: 5 items: type: string - order: 2 + order: 3 country: type: array description: 2-letter ISO 3166-1 countries (maximum 5) to restrict the search to. maxitems: 5 - order: 3 + order: 4 items: type: string enum: @@ -128,7 +134,7 @@ connectionSpecification: type: array description: Categories (maximum 5) to restrict the search to. maxitems: 5 - order: 4 + order: 5 items: type: string enum: @@ -147,7 +153,7 @@ connectionSpecification: type: array description: Languages (maximum 5) to restrict the search to. maxitems: 5 - order: 5 + order: 6 items: type: string enum: From fecbe1e1b2d2dcd2e2ec68bc6d0aca0b0d44743c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xabier=20Lahuerta=20V=C3=A1zquez?= Date: Tue, 1 Nov 2022 21:07:07 +0100 Subject: [PATCH 7/8] Remove `is_free_tier`, because as of now, dynamic `page_size` is not supported --- .../source_newsdata/newsdata.yaml | 2 +- .../source-newsdata/source_newsdata/spec.yaml | 22 +++++++------------ 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml b/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml index 1c897497fe6f..d20dd160dc78 100644 --- a/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml +++ b/airbyte-integrations/connectors/source-newsdata/source_newsdata/newsdata.yaml @@ -25,7 +25,7 @@ definitions: type: "CursorPagination" cursor_value: "{{ response['nextPage'] }}" page_size: 10 - # page_size: !!int "{% if config['is_free_tier'] %}{{ 10 }}{% else %}{{ 50 }}{% endif %}" + # TODO: make page_size dynamic, depending on free or paid tier. See https://github.com/airbytehq/airbyte/issues/18783 page_token_option: field_name: "page" inject_into: "request_parameter" diff --git a/airbyte-integrations/connectors/source-newsdata/source_newsdata/spec.yaml b/airbyte-integrations/connectors/source-newsdata/source_newsdata/spec.yaml index e7cb91011dd9..dd186f2a5c96 100644 --- a/airbyte-integrations/connectors/source-newsdata/source_newsdata/spec.yaml +++ b/airbyte-integrations/connectors/source-newsdata/source_newsdata/spec.yaml @@ -12,12 +12,6 @@ connectionSpecification: description: API Key airbyte_secret: true order: 0 - is_free_tier: - type: boolean - description: Indicates if API Key is free tier. This value affects page size, 10 if free tier, 50 if not. - default: true - airbyte_secret: true - order: 1 OneOf: query: type: string @@ -34,23 +28,23 @@ connectionSpecification: - Search `Social` or `pizza` but not `pasta`: query = "social OR pizza -pasta" - Search `Social` or `pizza` or `pasta`: query = "social OR pizza OR pasta" Note: You can't use AND and OR in the same query. - order: 2 + order: 1 query_in_title: type: string description: Same as `query`, but restricting the search to only the news title. It cannot be used along with `query`. - order: 2 + order: 1 domain: type: array description: Domains (maximum 5) to restrict the search to. Use the sources stream to find top sources id. maxitems: 5 items: type: string - order: 3 + order: 2 country: type: array description: 2-letter ISO 3166-1 countries (maximum 5) to restrict the search to. maxitems: 5 - order: 4 + order: 3 items: type: string enum: @@ -101,7 +95,7 @@ connectionSpecification: - nz - ng - kp - - no + - "no" - pk - pe - ph @@ -134,7 +128,7 @@ connectionSpecification: type: array description: Categories (maximum 5) to restrict the search to. maxitems: 5 - order: 5 + order: 4 items: type: string enum: @@ -153,7 +147,7 @@ connectionSpecification: type: array description: Languages (maximum 5) to restrict the search to. maxitems: 5 - order: 6 + order: 5 items: type: string enum: @@ -186,7 +180,7 @@ connectionSpecification: - lv - lt - ms - - no + - "no" - pl - pt - ro From 12225f19593344e742d38238cc71745a1fa9e781 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Thu, 10 Nov 2022 20:25:31 +0000 Subject: [PATCH 8/8] auto-bump connector version --- .../src/main/resources/seed/source_specs.yaml | 205 ++++++++++++++++++ 1 file changed, 205 insertions(+) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 51d5a7584a76..f0e11ba4a416 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -8848,6 +8848,211 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-newsdata:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/newsdata" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Newsdata Spec" + type: "object" + required: + - "api_key" + additionalProperties: true + properties: + api_key: + type: "string" + description: "API Key" + airbyte_secret: true + order: 0 + OneOf: + query: + type: "string" + description: "Keywords or phrases to search for in the news title and\ + \ content. Advanced Search options:\n - Search `Social`: query = \"\ + social\"\n - Search `Social Pizza`: query = \"social pizza\"\n - Search\ + \ `Social` but not with `pizza`: query = \"social -pizza\"\n - Search\ + \ `Social` but not with `pizza` and `wildfire`: query = \"social -pizza\ + \ -wildfire\"\n - Search `Social` and `pizza`: query = \"social AND\ + \ pizza\"\n - Search `Social` and `pizza` and `pasta`: query = \"social\ + \ AND pizza AND pasta\"\n - Search `Social` or `pizza`: query = \"\ + social OR pizza\"\n - Search `Social` or `pizza` but not `pasta`: query\ + \ = \"social OR pizza -pasta\"\n - Search `Social` or `pizza` or `pasta`:\ + \ query = \"social OR pizza OR pasta\"\nNote: You can't use AND and\ + \ OR in the same query." + order: 1 + query_in_title: + type: "string" + description: "Same as `query`, but restricting the search to only the\ + \ news title. It cannot be used along with `query`." + order: 1 + domain: + type: "array" + description: "Domains (maximum 5) to restrict the search to. Use the sources\ + \ stream to find top sources id." + maxitems: 5 + items: + type: "string" + order: 2 + country: + type: "array" + description: "2-letter ISO 3166-1 countries (maximum 5) to restrict the\ + \ search to." + maxitems: 5 + order: 3 + items: + type: "string" + enum: + - "ar" + - "au" + - "at" + - "bd" + - "by" + - "be" + - "br" + - "bg" + - "ca" + - "cl" + - "cn" + - "co" + - "cr" + - "cu" + - "cz" + - "dk" + - "do" + - "ec" + - "eg" + - "ee" + - "et" + - "fi" + - "fr" + - "de" + - "gr" + - "hk" + - "hu" + - "in" + - "id" + - "iq" + - "ie" + - "il" + - "it" + - "jp" + - "kz" + - "kw" + - "lv" + - "lb" + - "lt" + - "my" + - "mx" + - "ma" + - "mm" + - "nl" + - "nz" + - "ng" + - "kp" + - "no" + - "pk" + - "pe" + - "ph" + - "pl" + - "pt" + - "pr" + - "ro" + - "ru" + - "sa" + - "rs" + - "sg" + - "sk" + - "si" + - "za" + - "kr" + - "es" + - "se" + - "ch" + - "tw" + - "tz" + - "th" + - "tr" + - "ua" + - "ae" + - "gb" + - "us" + - "ve" + - "vi" + category: + type: "array" + description: "Categories (maximum 5) to restrict the search to." + maxitems: 5 + order: 4 + items: + type: "string" + enum: + - "business" + - "entertainment" + - "environment" + - "food" + - "health" + - "politics" + - "science" + - "sports" + - "technology" + - "top" + - "world" + language: + type: "array" + description: "Languages (maximum 5) to restrict the search to." + maxitems: 5 + order: 5 + items: + type: "string" + enum: + - "be" + - "am" + - "ar" + - "bn" + - "bs" + - "bg" + - "my" + - "ckb" + - "zh" + - "hr" + - "cs" + - "da" + - "nl" + - "en" + - "et" + - "fi" + - "fr" + - "de" + - "el" + - "he" + - "hi" + - "hu" + - "in" + - "it" + - "jp" + - "ko" + - "lv" + - "lt" + - "ms" + - "no" + - "pl" + - "pt" + - "ro" + - "ru" + - "sr" + - "sk" + - "sl" + - "es" + - "sw" + - "sv" + - "th" + - "tr" + - "uk" + - "ur" + - "vi" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-notion:0.1.10" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/notion"