Connector Generator: add additional pip install for tests in docs (#6411

)

* add additional pip install for tests

* run gradlew test scaffold

* correct flake and run format

* format generator files

* correct test_source scaffold gen

Co-authored-by: Marcos Marx <marcosmarx@MacBook-Pro-de-Marcos.local>

airbyte-integrations/connector-templates/source-python-http-api/README.md.hbs

@@ -21,6 +21,7 @@ development environment of choice. To activate it from the terminal, run:
 ```
 source .venv/bin/activate
 pip install -r requirements.txt
+pip install '.[tests]'
 ```
 If you are in an IDE, follow your IDE's instructions to activate the virtualenv.
 

airbyte-integrations/connector-templates/source-python-http-api/unit_tests/test_incremental_streams.py.hbs

@@ -23,7 +23,6 @@
 #
 
 from pytest import fixture
-from unittest.mock import MagicMock
 
 from airbyte_cdk.models import SyncMode
 from source_{{snakeCase name}}.source import Incremental{{properCase name}}Stream
@@ -46,7 +45,6 @@ def test_cursor_field(patch_incremental_base_class):
 
 def test_get_updated_state(patch_incremental_base_class):
     stream = Incremental{{properCase name}}Stream()
-    expected_cursor_field = []
     # TODO: replace this with your input parameters
     inputs = {"current_stream_state": None, "latest_record": None}
     # TODO: replace this with your expected updated stream state
@@ -56,7 +54,6 @@ def test_get_updated_state(patch_incremental_base_class):
 
 def test_stream_slices(patch_incremental_base_class):
     stream = Incremental{{properCase name}}Stream()
-    expected_cursor_field = []
     # TODO: replace this with your input parameters
     inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}}
     # TODO: replace this with your expected stream slices list
@@ -69,10 +66,12 @@ def test_supports_incremental(patch_incremental_base_class, mocker):
     stream = Incremental{{properCase name}}Stream()
     assert stream.supports_incremental
 
+
 def test_source_defined_cursor(patch_incremental_base_class):
     stream = Incremental{{properCase name}}Stream()
     assert stream.source_defined_cursor
 
+
 def test_stream_checkpoint_interval(patch_incremental_base_class):
     stream = Incremental{{properCase name}}Stream()
     # TODO: replace this with your expected checkpoint interval

airbyte-integrations/connector-templates/source-python-http-api/unit_tests/test_source.py.hbs

@@ -22,15 +22,17 @@
 # SOFTWARE.
 #
 
-from source_{{snakeCase name}}.source import Source{{properCase name}}
 from unittest.mock import MagicMock
 
+from source_{{snakeCase name}}.source import Source{{properCase name}}
+
 
 def test_check_connection(mocker):
     source = Source{{properCase name}}()
     logger_mock, config_mock = MagicMock(), MagicMock()
     assert source.check_connection(logger_mock, config_mock) == (True, None)
 
+
 def test_streams(mocker):
     source = Source{{properCase name}}()
     config_mock = MagicMock()

airbyte-integrations/connector-templates/source-python-http-api/unit_tests/test_streams.py.hbs

@@ -24,8 +24,8 @@
 
 from http import HTTPStatus
 from unittest.mock import MagicMock
-import pytest
 
+import pytest
 from source_{{snakeCase name}}.source import {{properCase name}}Stream
 
 
@@ -70,7 +70,7 @@ def test_request_headers(patch_base_class):
     inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None}
     # TODO: replace this with your expected request headers
     expected_headers = {}
-    assert stream.request_headers(**inputs) == {}
+    assert stream.request_headers(**inputs) == expected_headers
 
 
 def test_http_method(patch_base_class):

airbyte-integrations/connectors/source-scaffold-source-http/README.md

@@ -21,6 +21,7 @@ development environment of choice. To activate it from the terminal, run:
 ```
 source .venv/bin/activate
 pip install -r requirements.txt
+pip install '.[tests]'
 ```
 If you are in an IDE, follow your IDE's instructions to activate the virtualenv.
 

airbyte-integrations/connectors/source-scaffold-source-http/unit_tests/__init__.py

@@ -0,0 +1,23 @@
+#
+# MIT License
+#
+# Copyright (c) 2020 Airbyte
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#

airbyte-integrations/connectors/source-scaffold-source-http/unit_tests/test_incremental_streams.py

@@ -0,0 +1,79 @@
+#
+# MIT License
+#
+# Copyright (c) 2020 Airbyte
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+
+from pytest import fixture
+
+from airbyte_cdk.models import SyncMode
+from source_scaffold_source_http.source import IncrementalScaffoldSourceHttpStream
+
+
+@fixture
+def patch_incremental_base_class(mocker):
+    # Mock abstract methods to enable instantiating abstract class
+    mocker.patch.object(IncrementalScaffoldSourceHttpStream, "path", "v0/example_endpoint")
+    mocker.patch.object(IncrementalScaffoldSourceHttpStream, "primary_key", "test_primary_key")
+    mocker.patch.object(IncrementalScaffoldSourceHttpStream, "__abstractmethods__", set())
+
+
+def test_cursor_field(patch_incremental_base_class):
+    stream = IncrementalScaffoldSourceHttpStream()
+    # TODO: replace this with your expected cursor field
+    expected_cursor_field = []
+    assert stream.cursor_field == expected_cursor_field
+
+
+def test_get_updated_state(patch_incremental_base_class):
+    stream = IncrementalScaffoldSourceHttpStream()
+    # TODO: replace this with your input parameters
+    inputs = {"current_stream_state": None, "latest_record": None}
+    # TODO: replace this with your expected updated stream state
+    expected_state = {}
+    assert stream.get_updated_state(**inputs) == expected_state
+
+
+def test_stream_slices(patch_incremental_base_class):
+    stream = IncrementalScaffoldSourceHttpStream()
+    # TODO: replace this with your input parameters
+    inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}}
+    # TODO: replace this with your expected stream slices list
+    expected_stream_slice = [None]
+    assert stream.stream_slices(**inputs) == expected_stream_slice
+
+
+def test_supports_incremental(patch_incremental_base_class, mocker):
+    mocker.patch.object(IncrementalScaffoldSourceHttpStream, "cursor_field", "dummy_field")
+    stream = IncrementalScaffoldSourceHttpStream()
+    assert stream.supports_incremental
+
+
+def test_source_defined_cursor(patch_incremental_base_class):
+    stream = IncrementalScaffoldSourceHttpStream()
+    assert stream.source_defined_cursor
+
+
+def test_stream_checkpoint_interval(patch_incremental_base_class):
+    stream = IncrementalScaffoldSourceHttpStream()
+    # TODO: replace this with your expected checkpoint interval
+    expected_checkpoint_interval = None
+    assert stream.state_checkpoint_interval == expected_checkpoint_interval

airbyte-integrations/connectors/source-scaffold-source-http/unit_tests/test_source.py

@@ -0,0 +1,42 @@
+#
+# MIT License
+#
+# Copyright (c) 2020 Airbyte
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+
+from unittest.mock import MagicMock
+
+from source_scaffold_source_http.source import SourceScaffoldSourceHttp
+
+
+def test_check_connection(mocker):
+    source = SourceScaffoldSourceHttp()
+    logger_mock, config_mock = MagicMock(), MagicMock()
+    assert source.check_connection(logger_mock, config_mock) == (True, None)
+
+
+def test_streams(mocker):
+    source = SourceScaffoldSourceHttp()
+    config_mock = MagicMock()
+    streams = source.streams(config_mock)
+    # TODO: replace this with your streams number
+    expected_streams_number = 2
+    assert len(streams) == expected_streams_number

airbyte-integrations/connectors/source-scaffold-source-http/unit_tests/test_streams.py

@@ -0,0 +1,103 @@
+#
+# MIT License
+#
+# Copyright (c) 2020 Airbyte
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+
+from http import HTTPStatus
+from unittest.mock import MagicMock
+
+import pytest
+from source_scaffold_source_http.source import ScaffoldSourceHttpStream
+
+
+@pytest.fixture
+def patch_base_class(mocker):
+    # Mock abstract methods to enable instantiating abstract class
+    mocker.patch.object(ScaffoldSourceHttpStream, "path", "v0/example_endpoint")
+    mocker.patch.object(ScaffoldSourceHttpStream, "primary_key", "test_primary_key")
+    mocker.patch.object(ScaffoldSourceHttpStream, "__abstractmethods__", set())
+
+
+def test_request_params(patch_base_class):
+    stream = ScaffoldSourceHttpStream()
+    # TODO: replace this with your input parameters
+    inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None}
+    # TODO: replace this with your expected request parameters
+    expected_params = {}
+    assert stream.request_params(**inputs) == expected_params
+
+
+def test_next_page_token(patch_base_class):
+    stream = ScaffoldSourceHttpStream()
+    # TODO: replace this with your input parameters
+    inputs = {"response": MagicMock()}
+    # TODO: replace this with your expected next page token
+    expected_token = None
+    assert stream.next_page_token(**inputs) == expected_token
+
+
+def test_parse_response(patch_base_class):
+    stream = ScaffoldSourceHttpStream()
+    # TODO: replace this with your input parameters
+    inputs = {"response": MagicMock()}
+    # TODO: replace this with your expected parced object
+    expected_parsed_object = {}
+    assert next(stream.parse_response(**inputs)) == expected_parsed_object
+
+
+def test_request_headers(patch_base_class):
+    stream = ScaffoldSourceHttpStream()
+    # TODO: replace this with your input parameters
+    inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None}
+    # TODO: replace this with your expected request headers
+    expected_headers = {}
+    assert stream.request_headers(**inputs) == expected_headers
+
+
+def test_http_method(patch_base_class):
+    stream = ScaffoldSourceHttpStream()
+    # TODO: replace this with your expected http request method
+    expected_method = "GET"
+    assert stream.http_method == expected_method
+
+
+@pytest.mark.parametrize(
+    ("http_status", "should_retry"),
+    [
+        (HTTPStatus.OK, False),
+        (HTTPStatus.BAD_REQUEST, False),
+        (HTTPStatus.TOO_MANY_REQUESTS, True),
+        (HTTPStatus.INTERNAL_SERVER_ERROR, True),
+    ],
+)
+def test_should_retry(patch_base_class, http_status, should_retry):
+    response_mock = MagicMock()
+    response_mock.status_code = http_status
+    stream = ScaffoldSourceHttpStream()
+    assert stream.should_retry(response_mock) == should_retry
+
+
+def test_backoff_time(patch_base_class):
+    response_mock = MagicMock()
+    stream = ScaffoldSourceHttpStream()
+    expected_backoff_time = None
+    assert stream.backoff_time(response_mock) == expected_backoff_time