diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 92539c2da9e6..2421c252645e 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -881,7 +881,7 @@ - name: Short.io sourceDefinitionId: 2fed2292-5586-480c-af92-9944e39fe12d dockerRepository: airbyte/source-shortio - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/sources/shortio icon: short.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 49cfcb0b65c4..d8123a31d7d2 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -8592,7 +8592,7 @@ path_in_connector_config: - "credentials" - "client_secret" -- dockerImage: "airbyte/source-shortio:0.1.2" +- dockerImage: "airbyte/source-shortio:0.1.3" spec: documentationUrl: "https://developers.short.io/reference" connectionSpecification: @@ -8603,7 +8603,6 @@ - "domain_id" - "secret_key" - "start_date" - additionalProperties: false properties: domain_id: type: "string" diff --git a/airbyte-integrations/connectors/source-shortio/Dockerfile b/airbyte-integrations/connectors/source-shortio/Dockerfile index 7d0b19c57377..33aa353896c3 100644 --- a/airbyte-integrations/connectors/source-shortio/Dockerfile +++ b/airbyte-integrations/connectors/source-shortio/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/source-shortio diff --git a/airbyte-integrations/connectors/source-shortio/README.md b/airbyte-integrations/connectors/source-shortio/README.md index 74b36cdfb230..11376313b52c 100644 --- a/airbyte-integrations/connectors/source-shortio/README.md +++ b/airbyte-integrations/connectors/source-shortio/README.md @@ -8,7 +8,7 @@ For information about how to use this connector within Airbyte, see [the documen ### Prerequisites **To iterate on this connector, make sure to complete this prerequisites section.** -#### Minimum Python version required `= 3.7.0` +#### Minimum Python version required `= 3.9.0` #### Build & Activate Virtual Environment and install dependencies From this connector directory, create a virtual environment: diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-shortio/integration_tests/abnormal_state.json index b57ea413e847..b5425313f16e 100644 --- a/airbyte-integrations/connectors/source-shortio/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-shortio/integration_tests/abnormal_state.json @@ -1,5 +1,5 @@ { "clicks": { - "dt": "2022-07-17 14:03:43.449925" + "dt": "2052-07-17 14:03:43.449925" } } diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/catalog.json b/airbyte-integrations/connectors/source-shortio/integration_tests/catalog.json deleted file mode 100644 index 6799946a6851..000000000000 --- a/airbyte-integrations/connectors/source-shortio/integration_tests/catalog.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "streams": [ - { - "name": "TODO fix this file", - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": "column1", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "column1": { - "type": "string" - }, - "column2": { - "type": "number" - } - } - } - }, - { - "name": "table1", - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "column1": { - "type": "string" - }, - "column2": { - "type": "number" - } - } - } - } - ] -} diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-shortio/integration_tests/configured_catalog.json index 1d403515b60d..025b5475ee2f 100644 --- a/airbyte-integrations/connectors/source-shortio/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-shortio/integration_tests/configured_catalog.json @@ -8,83 +8,7 @@ "source_defined_cursor": true, "default_cursor_field": ["dt"], "supported_sync_modes": ["incremental"], - "json_schema": { - "properties": { - "host": { - "type": ["null", "string"] - }, - "path": { - "type": ["null", "string"] - }, - "method": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - }, - "dt": { - "type": ["null", "string"], - "format": "date-time" - }, - "st": { - "type": ["null", "integer"] - }, - "ip": { - "type": ["null", "string"] - }, - "proto": { - "type": ["null", "string"] - }, - "ref": { - "type": ["null", "string"] - }, - "ua": { - "type": ["null", "string"] - }, - "human": { - "type": ["null", "boolean"] - }, - "browser": { - "type": ["null", "string"] - }, - "browser_version": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "social": { - "type": ["null", "string"] - }, - "refhost": { - "type": ["null", "string"] - }, - "os": { - "type": ["null", "string"] - }, - "utm_source": { - "type": ["null", "string"] - }, - "utm_medium": { - "type": ["null", "string"] - }, - "utm_campaign": { - "type": ["null", "string"] - }, - "goal_completed": { - "type": ["null", "string"] - }, - "ab_path": { - "type": ["null", "string"] - }, - "lcpath": { - "type": ["null", "string"] - } - } - } + "json_schema": {} } }, { @@ -94,155 +18,7 @@ "name": "links", "source_defined_cursor": true, "supported_sync_modes": ["full_refresh"], - "json_schema": { - "properties": { - "id": { - "type": "integer" - }, - "path": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "icon": { - "type": ["null", "string"] - }, - "archived": { - "type": ["null", "boolean"] - }, - "originalURL": { - "type": "string" - }, - "iphoneURL": { - "type": ["null", "string"] - }, - "androidURL": { - "type": ["null", "string"] - }, - "password": { - "type": ["null", "string"] - }, - "utmSource": { - "type": ["null", "string"] - }, - "utmMedium": { - "type": ["null", "string"] - }, - "utmCampaign": { - "type": ["null", "string"] - }, - "utmCampaignId": { - "type": ["null", "string"] - }, - "utmTerm": { - "type": ["null", "string"] - }, - "utmContent": { - "type": ["null", "string"] - }, - "splitURL": { - "type": ["null", "string"] - }, - "splitPercent": { - "type": ["null", "string"] - }, - "expiresAt": { - "type": ["null", "string"] - }, - "expiredURL": { - "type": ["null", "string"] - }, - "redirectType": { - "type": ["null", "string"] - }, - "clicksLimit": { - "type": ["null", "string"] - }, - "cloaking": { - "type": ["null", "boolean"] - }, - "source": { - "type": ["null", "string"] - }, - "integrationGA": { - "type": ["null", "string"] - }, - "integrationFB": { - "type": ["null", "string"] - }, - "integrationAdroll": { - "type": ["null", "string"] - }, - "integrationGTM": { - "type": ["null", "string"] - }, - "AutodeletedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "createdAt": { - "type": "string", - "format": "date-time" - }, - "updatedAt": { - "type": "string", - "format": "date-time" - }, - "DomainId": { - "type": "integer" - }, - "Owner": { - "type": ["null", "object"], - "properties": { - "id": { - "type": "integer" - }, - "name": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "photoURL": { - "type": ["null", "string"] - } - } - }, - "tags": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "secureShortURL": { - "type": ["null", "string"] - }, - "idString": { - "type": "string" - }, - "shortURL": { - "type": "string" - }, - "User": { - "type": ["null", "object"], - "properties": { - "id": { - "type": "integer" - }, - "name": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "photoURL": { - "type": ["null", "string"] - } - } - } - } - } + "json_schema": {} } } ] diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-shortio/integration_tests/sample_config.json deleted file mode 100644 index ecc4913b84c7..000000000000 --- a/airbyte-integrations/connectors/source-shortio/integration_tests/sample_config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "fix-me": "TODO" -} diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/state.json b/airbyte-integrations/connectors/source-shortio/integration_tests/state.json index 85ee259630df..322d3ab704a4 100644 --- a/airbyte-integrations/connectors/source-shortio/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-shortio/integration_tests/state.json @@ -1,5 +1,5 @@ { "clicks": { - "dt": "2021-07-28 14:03:43.449925" + "dt": "2022-07-27 10:30:43.449925" } } diff --git a/airbyte-integrations/connectors/source-shortio/setup.py b/airbyte-integrations/connectors/source-shortio/setup.py index 102830d0836a..afd147bfd877 100644 --- a/airbyte-integrations/connectors/source-shortio/setup.py +++ b/airbyte-integrations/connectors/source-shortio/setup.py @@ -5,12 +5,10 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", -] +MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1.56"] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "pytest~=6.2.5", "source-acceptance-test", ] diff --git a/airbyte-integrations/connectors/source-shortio/source_shortio/schemas/links.json b/airbyte-integrations/connectors/source-shortio/source_shortio/schemas/links.json index 6a55f8eb929d..ae204f7aa53f 100644 --- a/airbyte-integrations/connectors/source-shortio/source_shortio/schemas/links.json +++ b/airbyte-integrations/connectors/source-shortio/source_shortio/schemas/links.json @@ -2,9 +2,6 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "properties": { - "id": { - "type": "integer" - }, "path": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-shortio/source_shortio/source.py b/airbyte-integrations/connectors/source-shortio/source_shortio/source.py index 8ae6e5386958..12dd740eb489 100644 --- a/airbyte-integrations/connectors/source-shortio/source_shortio/source.py +++ b/airbyte-integrations/connectors/source-shortio/source_shortio/source.py @@ -3,6 +3,7 @@ # +import contextlib import datetime import json from abc import ABC @@ -24,7 +25,7 @@ class Links(HttpStream, ABC): url_base = "https://api.short.io/api/" limit = 150 - primary_key = "id" + primary_key = "idString" before_id = None domain_id = None @@ -77,18 +78,16 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp "utmContent": "utm_content", } links = json.loads(response.text)["links"] - for index, item in enumerate(links): + for item in links: for resp_field, param in utm_response_fields_to_utm_params.items(): if resp_field not in item.keys(): - param = param + "=" + param = f"{param}=" original_url = item["originalURL"] param_value = None - try: + with contextlib.suppress(IndexError): # Extracting parameter value from original URL # i.e "talent" from http://airbyte.io/?utm_source=talent param_value = original_url.split(param, 2)[1].split("&", 1)[0] - except IndexError: - pass item[resp_field] = param_value yield item @@ -142,10 +141,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, clicks = json.loads(response.text) try: before_dt = sorted(clicks, key=lambda k: k["dt"], reverse=False)[0]["dt"] - if self.limit > len(clicks): - return None - else: - return before_dt + return None if self.limit > len(clicks) else before_dt except IndexError: return None @@ -181,19 +177,13 @@ def request_body_json( :return dict: json body for the request """ - payload = { + return { "limit": self.limit, "include": {"human": True}, "beforeDate": next_page_token or self.before_dt, + "afterDate": stream_state["dt"] if stream_state and "dt" in stream_state.keys() else self.start_date, } - if stream_state and "dt" in stream_state.keys(): - payload["afterDate"] = stream_state["dt"] - else: - payload["afterDate"] = self.start_date - - return payload - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: yield from json.loads(response.text) diff --git a/airbyte-integrations/connectors/source-shortio/source_shortio/spec.json b/airbyte-integrations/connectors/source-shortio/source_shortio/spec.json index 37845571418f..27e39c4a96ef 100644 --- a/airbyte-integrations/connectors/source-shortio/source_shortio/spec.json +++ b/airbyte-integrations/connectors/source-shortio/source_shortio/spec.json @@ -5,7 +5,6 @@ "title": "Shortio Spec", "type": "object", "required": ["domain_id", "secret_key", "start_date"], - "additionalProperties": false, "properties": { "domain_id": { "type": "string", diff --git a/airbyte-integrations/connectors/source-shortio/unit_tests/test_source.py b/airbyte-integrations/connectors/source-shortio/unit_tests/test_source.py new file mode 100644 index 000000000000..8353c88ea7b2 --- /dev/null +++ b/airbyte-integrations/connectors/source-shortio/unit_tests/test_source.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import pytest +from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.models import Status +from source_shortio.source import SourceShortio + + +@pytest.fixture +def config(): + return {"domain_id": "foo", "secret_key": "bar", "start_date": "2030-01-01"} + + +def test_source_shortio_client_wrong_credentials(): + source = SourceShortio() + result = source.check(logger=AirbyteLogger, config={"domain_id": "foo", "secret_key": "bar", "start_date": "2030-01-01"}) + assert result.status == Status.FAILED + + +def test_streams(): + source = SourceShortio() + config_mock = {"domain_id": "foo", "secret_key": "bar", "start_date": "2030-01-01"} + streams = source.streams(config_mock) + expected_streams_number = 2 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-shortio/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-shortio/unit_tests/unit_test.py deleted file mode 100644 index dddaea0060fa..000000000000 --- a/airbyte-integrations/connectors/source-shortio/unit_tests/unit_test.py +++ /dev/null @@ -1,7 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - - -def test_example_method(): - assert True diff --git a/docs/integrations/sources/shortio.md b/docs/integrations/sources/shortio.md index 1d56fd51aa6f..27a9a0b43a52 100644 --- a/docs/integrations/sources/shortio.md +++ b/docs/integrations/sources/shortio.md @@ -43,6 +43,7 @@ This Source is capable of syncing the following Streams: | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.3 | 2022-08-01 | [15066](https://github.com/airbytehq/airbyte/pull/15066) | Update primary key to `idString` | | 0.1.2 | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628) | Update fields in source-connectors specifications | | 0.1.1 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | | 0.1.0 | 2021-08-16 | [3787](https://github.com/airbytehq/airbyte/pull/5418) | Add Native Shortio Source Connector |