🎉Source Instagram: have at least 90% unit test coverage (#11260)

* Add new unittests

* Increased unit test coverage to 90

* Fixed to linter

* Updated doc

* Updated README.md

airbyte-integrations/connectors/source-instagram/README.md

@@ -59,6 +59,21 @@ To run unit tests locally, from the connector directory run:
 python -m pytest unit_tests
 ```
 
+#### Acceptance Tests
+Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information.
+If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
+To run your integration tests with acceptance tests, from the connector root, run
+```
+docker build . --no-cache -t airbyte/source-instagram:dev \
+&& python -m pytest -p source_acceptance_test.plugin
+```
+or
+```
+./acceptance-test-docker.sh 
+```
+
+To run your integration tests with docker
+
 ### Locally running the connector docker image
 
 #### Build

airbyte-integrations/connectors/source-instagram/setup.py

@@ -15,7 +15,8 @@
 
 TEST_REQUIREMENTS = [
     "pytest~=6.1",
-    "requests_mock==1.8.0",
+    "pytest-mock~=3.6",
+    "requests_mock~=1.8",
 ]
 
 setup(

airbyte-integrations/connectors/source-instagram/source_instagram/common.py

@@ -7,10 +7,12 @@
 import urllib.parse as urlparse
 
 import backoff
-from airbyte_cdk.entrypoint import logger  # FIXME (Eugene K): register logger as standard python logger
+from airbyte_cdk.logger import AirbyteLogger
 from facebook_business.exceptions import FacebookRequestError
 from requests.status_codes import codes as status_codes
 
+logger = AirbyteLogger()
+
 
 class InstagramAPIException(Exception):
     """General class for all API errors"""
@@ -38,9 +40,12 @@ def should_retry_api_error(exc: FacebookRequestError):
         if exc.http_status() == status_codes.TOO_MANY_REQUESTS:
             return True
 
-        # FIXME: add type and http_status
-        if exc.api_error_code() == 10 and exc.api_error_message() == "(#10) Not enough viewers for the media to show insights":
-            return False  # expected error
+        if (
+            exc.api_error_type() == "OAuthException"
+            and exc.api_error_code() == 10
+            and exc.api_error_message() == "(#10) Not enough viewers for the media to show insights"
+        ):
+            return True
 
         # Issue 4028, Sometimes an error about the Rate Limit is returned with a 400 HTTP code
         if exc.http_status() == status_codes.BAD_REQUEST and exc.api_error_code() == 100 and exc.api_error_subcode() == 33:
@@ -49,9 +54,10 @@ def should_retry_api_error(exc: FacebookRequestError):
         if exc.api_transient_error():
             return True
 
-        # FIXME: add type, code and http_status
-        if exc.api_error_subcode() == 2108006:
-            return False
+        # The media was posted before the most recent time that the user's account
+        # was converted to a business account from a personal account.
+        if exc.api_error_type() == "OAuthException" and exc.api_error_code() == 100 and exc.api_error_subcode() == 2108006:
+            return True
 
         return False
 
@@ -66,12 +72,9 @@ def should_retry_api_error(exc: FacebookRequestError):
 
 
 def remove_params_from_url(url, params):
-    parsed_url = urlparse.urlparse(url)
-    res_query = []
-    for q in parsed_url.query.split("&"):
-        key, value = q.split("=")
-        if key not in params:
-            res_query.append(f"{key}={value}")
-
-    parse_result = parsed_url._replace(query="&".join(res_query))
-    return urlparse.urlunparse(parse_result)
+    parsed = urlparse.urlparse(url)
+    query = urlparse.parse_qs(parsed.query, keep_blank_values=True)
+    filtered = dict((k, v) for k, v in query.items() if k not in params)
+    return urlparse.urlunparse(
+        [parsed.scheme, parsed.netloc, parsed.path, parsed.params, urlparse.urlencode(filtered, doseq=True), parsed.fragment]
+    )

airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py

@@ -0,0 +1,104 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+from facebook_business import FacebookAdsApi, FacebookSession
+from pytest import fixture
+from source_instagram.api import InstagramAPI as API
+
+FB_API_VERSION = FacebookAdsApi.API_VERSION
+
+
+@fixture(scope="session", name="account_id")
+def account_id_fixture():
+    return "unknown_account"
+
+
+@fixture(name="config")
+def config_fixture():
+    config = {
+        "access_token": "TOKEN",
+        "start_date": "2022-03-20T00:00:00",
+    }
+
+    return config
+
+
+@fixture(scope="session", name="some_config")
+def some_config_fixture(account_id):
+    return {"start_date": "2021-01-23T00:00:00Z", "access_token": "unknown_token"}
+
+
+@fixture(name="fb_account_response")
+def fb_account_response_fixture(account_id, some_config, requests_mock):
+    account = {"id": "test_id", "instagram_business_account": {"id": "test_id"}}
+    requests_mock.register_uri(
+        "GET",
+        FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/"
+        f"?access_token={some_config['access_token']}&fields=instagram_business_account",
+        json=account,
+    )
+    return {
+        "json": {
+            "data": [
+                {
+                    "account_id": account_id,
+                    "id": f"act_{account_id}",
+                }
+            ],
+            "paging": {"cursors": {"before": "MjM4NDYzMDYyMTcyNTAwNzEZD", "after": "MjM4NDYzMDYyMTcyNTAwNzEZD"}},
+        },
+        "status_code": 200,
+    }
+
+
+@fixture(name="api")
+def api_fixture(some_config, requests_mock, fb_account_response):
+    api = API(access_token=some_config["access_token"])
+
+    requests_mock.register_uri(
+        "GET",
+        FacebookSession.GRAPH + f"/{FB_API_VERSION}/me/accounts?" f"access_token={some_config['access_token']}&summary=true",
+        [fb_account_response],
+    )
+
+    return api
+
+
+@fixture(name="user_data")
+def user_data_fixture():
+    return {
+        "biography": "Dino data crunching app",
+        "id": "17841405822304914",
+        "username": "metricsaurus",
+        "website": "http://www.metricsaurus.com/",
+    }
+
+
+@fixture(name="user_insight_data")
+def user_insight_data_fixture():
+    return {
+        "name": "impressions",
+        "period": "day",
+        "values": [{"value": 4, "end_time": "2020-05-04T07:00:00+0000"}, {"value": 66, "end_time": "2020-05-05T07:00:00+0000"}],
+        "title": "Impressions",
+        "description": "Total number of times this profile has been seen",
+        "id": "17841400008460056/insights/impressions/day",
+    }
+
+
+@fixture(name="user_stories_data")
+def user_stories_data_fixture():
+    return {"id": "test_id"}
+
+
+@fixture(name="user_media_insights_data")
+def user_media_insights_data_fixture():
+    return {
+        "name": "impressions",
+        "period": "lifetime",
+        "values": [{"value": 264}],
+        "title": "Impressions",
+        "description": "Total number of times the media object has been seen",
+        "id": "17855590849148465/insights/impressions/lifetime",
+    }

airbyte-integrations/connectors/source-instagram/unit_tests/test_common.py

@@ -0,0 +1,53 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+from source_instagram.common import remove_params_from_url
+
+
+def test_empty_url():
+    url = ""
+    parsed_url = remove_params_from_url(url=url, params=[])
+    assert parsed_url == url
+
+
+def test_does_not_raise_exception_for_invalid_url():
+    url = "abcd"
+    parsed_url = remove_params_from_url(url=url, params=["test"])
+    assert parsed_url == url
+
+
+def test_escaped_characters():
+    url = "https://google.com?test=123%23%24%25%2A&test2=456"
+    parsed_url = remove_params_from_url(url=url, params=["test3"])
+    assert parsed_url == url
+
+
+def test_no_params_url():
+    url = "https://google.com"
+    parsed_url = remove_params_from_url(url=url, params=["test"])
+    assert parsed_url == url
+
+
+def test_no_params_arg():
+    url = "https://google.com?"
+    parsed_url = remove_params_from_url(url=url, params=["test"])
+    assert parsed_url == "https://google.com"
+
+
+def test_partially_empty_params():
+    url = "https://google.com?test=122&&"
+    parsed_url = remove_params_from_url(url=url, params=[])
+    assert parsed_url == "https://google.com?test=122"
+
+
+def test_no_matching_params():
+    url = "https://google.com?test=123"
+    parsed_url = remove_params_from_url(url=url, params=["test2"])
+    assert parsed_url == url
+
+
+def test_removes_params():
+    url = "https://google.com?test=123&test2=456"
+    parsed_url = remove_params_from_url(url=url, params=["test2"])
+    assert parsed_url == "https://google.com?test=123"

airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py

@@ -0,0 +1,74 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+
+from airbyte_cdk.logger import AirbyteLogger
+from airbyte_cdk.models import (
+    AirbyteStream,
+    ConfiguredAirbyteCatalog,
+    ConfiguredAirbyteStream,
+    ConnectorSpecification,
+    DestinationSyncMode,
+    SyncMode,
+)
+from source_instagram.source import SourceInstagram
+
+logger = AirbyteLogger()
+
+
+def test_check_connection_ok(api, some_config):
+    ok, error_msg = SourceInstagram().check_connection(logger, config=some_config)
+
+    assert ok
+    assert not error_msg
+
+
+def test_check_connection_empty_config(api):
+    config = {}
+    ok, error_msg = SourceInstagram().check_connection(logger, config=config)
+
+    assert not ok
+    assert error_msg
+
+
+def test_check_connection_invalid_config(api, some_config):
+    some_config.pop("start_date")
+    ok, error_msg = SourceInstagram().check_connection(logger, config=some_config)
+
+    assert not ok
+    assert error_msg
+
+
+def test_check_connection_exception(api, config):
+    api.side_effect = RuntimeError("Something went wrong!")
+    ok, error_msg = SourceInstagram().check_connection(logger, config=config)
+
+    assert not ok
+    assert error_msg
+
+
+def test_streams(api, config):
+    streams = SourceInstagram().streams(config)
+
+    assert len(streams) == 7
+
+
+def test_spec():
+    spec = SourceInstagram().spec()
+
+    assert isinstance(spec, ConnectorSpecification)
+
+
+def test_read(config):
+    source = SourceInstagram()
+    catalog = ConfiguredAirbyteCatalog(
+        streams=[
+            ConfiguredAirbyteStream(
+                stream=AirbyteStream(name="users", json_schema={}),
+                sync_mode=SyncMode.full_refresh,
+                destination_sync_mode=DestinationSyncMode.overwrite,
+            )
+        ]
+    )
+    assert source.read(logger, config, catalog)