From 2405054b8748d4be2d45236e1828629f311dad2d Mon Sep 17 00:00:00 2001 From: namrata270998 Date: Tue, 16 Nov 2021 05:50:57 +0000 Subject: [PATCH 01/15] added request timeout --- tap_dynamodb/discover.py | 8 +- tap_dynamodb/dynamodb.py | 36 +++++- tap_dynamodb/sync_strategies/full_table.py | 7 +- tap_dynamodb/sync_strategies/log_based.py | 13 +- tests/unittests/test_request_timeout.py | 142 +++++++++++++++++++++ 5 files changed, 196 insertions(+), 10 deletions(-) create mode 100644 tests/unittests/test_request_timeout.py diff --git a/tap_dynamodb/discover.py b/tap_dynamodb/discover.py index 130a9e7..c4b9534 100644 --- a/tap_dynamodb/discover.py +++ b/tap_dynamodb/discover.py @@ -2,6 +2,8 @@ import singer from botocore.exceptions import ClientError from tap_dynamodb import dynamodb +import backoff +from botocore.exceptions import ClientError, ConnectTimeoutError, ReadTimeoutError LOGGER = singer.get_logger() @@ -29,7 +31,11 @@ def discover_table_schema(client, table_name): } } - +# Backoff for both ReadTimeout and ConnectTimeout error for 5 times +@backoff.on_exception(backoff.expo, + (ReadTimeoutError, ConnectTimeoutError), + max_tries=5, + factor=2) def discover_streams(config): client = dynamodb.get_client(config) diff --git a/tap_dynamodb/dynamodb.py b/tap_dynamodb/dynamodb.py index e503b2b..5944024 100644 --- a/tap_dynamodb/dynamodb.py +++ b/tap_dynamodb/dynamodb.py @@ -8,14 +8,16 @@ DeferredRefreshableCredentials, JSONFileCache ) -from botocore.exceptions import ClientError from botocore.session import Session +from botocore.config import Config +from botocore.exceptions import ClientError, ConnectTimeoutError, ReadTimeoutError LOGGER = singer.get_logger() +REQUEST_TIMEOUT = 300 def retry_pattern(): return backoff.on_exception(backoff.expo, - ClientError, + (ClientError, ConnectTimeoutError, ReadTimeoutError), max_tries=5, on_backoff=log_backoff_attempt, factor=10) @@ -66,16 +68,38 @@ def setup_aws_client(config): boto3.setup_default_session(botocore_session=refreshable_session) def get_client(config): + # if request_timeout is other than 0,"0" or "" then use request_timeout + request_timeout = config.get('request_timeout') + if request_timeout and float(request_timeout): + request_timeout = float(request_timeout) + else: # If value is 0,"0" or "" then set default to 300 seconds. + request_timeout = REQUEST_TIMEOUT + timeout_config = Config(connect_timeout=request_timeout, read_timeout=request_timeout) if config.get('use_local_dynamo'): return boto3.client('dynamodb', endpoint_url='http://localhost:8000', - region_name=config['region_name']) - return boto3.client('dynamodb', config['region_name']) + region_name=config['region_name'], + config=timeout_config + ) + return boto3.client('dynamodb', config['region_name'], + config=timeout_config + ) def get_stream_client(config): + # if request_timeout is other than 0,"0" or "" then use request_timeout + request_timeout = config.get('request_timeout') + if request_timeout and float(request_timeout): + request_timeout = float(request_timeout) + else: # If value is 0,"0" or "" then set default to 300 seconds. + request_timeout = REQUEST_TIMEOUT + timeout_config = Config(connect_timeout=request_timeout, read_timeout=request_timeout) if config.get('use_local_dynamo'): return boto3.client('dynamodbstreams', endpoint_url='http://localhost:8000', - region_name=config['region_name']) + region_name=config['region_name'], + config=timeout_config + ) return boto3.client('dynamodbstreams', - region_name=config['region_name']) + region_name=config['region_name'], + config=timeout_config + ) diff --git a/tap_dynamodb/sync_strategies/full_table.py b/tap_dynamodb/sync_strategies/full_table.py index 7aa4dfc..e0c9fd1 100644 --- a/tap_dynamodb/sync_strategies/full_table.py +++ b/tap_dynamodb/sync_strategies/full_table.py @@ -4,10 +4,15 @@ from singer import metadata from tap_dynamodb import dynamodb from tap_dynamodb.deserialize import Deserializer +import backoff +from botocore.exceptions import ClientError, ConnectTimeoutError, ReadTimeoutError LOGGER = singer.get_logger() - +@backoff.on_exception(backoff.expo, + (ReadTimeoutError, ConnectTimeoutError), + max_tries=5, + factor=2) def scan_table(table_name, projection, last_evaluated_key, config): scan_params = { 'TableName': table_name, diff --git a/tap_dynamodb/sync_strategies/log_based.py b/tap_dynamodb/sync_strategies/log_based.py index 4ac37e0..a05fc8f 100644 --- a/tap_dynamodb/sync_strategies/log_based.py +++ b/tap_dynamodb/sync_strategies/log_based.py @@ -4,6 +4,8 @@ from tap_dynamodb import dynamodb from tap_dynamodb import deserialize +import backoff +from botocore.exceptions import ConnectTimeoutError, ReadTimeoutError LOGGER = singer.get_logger() WRITE_STATE_PERIOD = 1000 @@ -115,7 +117,11 @@ def sync_shard(shard, seq_number_bookmarks, streams_client, stream_arn, projecti singer.write_state(state) return rows_synced - +# Backoff for both ReadTimeout and ConnectTimeout error for 5 times +@backoff.on_exception(backoff.expo, + (ReadTimeoutError, ConnectTimeoutError), + max_tries=5, + factor=2) def sync(config, state, stream): table_name = stream['tap_stream_id'] @@ -212,7 +218,10 @@ def has_stream_aged_out(state, table_name): # stream then we consider the stream to be aged out return time_span > datetime.timedelta(hours=19, minutes=30) - +@backoff.on_exception(backoff.expo, + (ReadTimeoutError, ConnectTimeoutError), + max_tries=5, + factor=2) def get_initial_bookmarks(config, state, table_name): ''' Returns the state including all bookmarks necessary for the initial diff --git a/tests/unittests/test_request_timeout.py b/tests/unittests/test_request_timeout.py new file mode 100644 index 0000000..c3d0056 --- /dev/null +++ b/tests/unittests/test_request_timeout.py @@ -0,0 +1,142 @@ +from tap_dynamodb.sync_strategies import full_table, log_based +from tap_dynamodb import discover, dynamodb +import tap_dynamodb +import unittest +from unittest import mock +from unittest.mock import Mock +from unittest.case import TestCase +from botocore.exceptions import ClientError, ConnectTimeoutError, ReadTimeoutError + +class MockClient(): + def __init__(self, endpoint_url=None): + pass + + def list_tables(self): + pass + + + +class TestBackoffError(unittest.TestCase): + ''' + Test that backoff logic works properly. + ''' + # @mock.patch('tap_dynamodb.dynamodb.boto3.client', return_value=MockClient()) + @mock.patch('boto3.client') + @mock.patch('singer.metadata.to_map') + def test_discover_stream_request_timeout_and_backoff(self, mock_map, mock_client): + """ + Check whether the request backoffs properly for list_tables for 5 times in case of Timeout error. + """ + mock_client.list_tables.side_effect = ReadTimeoutError + # mock_client = Mock() + # mock_client.list_tables = Mock() + + with self.assertRaises(ReadTimeoutError): + # log_based.sync({"region_name": "us-east-2"}, {}, {"tap_stream_id": "dummy", "metadata":""}) + discover.discover_streams({"region_name": "dummy", "use_local_dynamo": "true"}) + self.assertEquals(mock_client.list_tables.call_count, 5) + + +class TestRequestTimeoutValue(unittest.TestCase): + ''' + Test that request timeout parameter works properly in various cases + ''' + @mock.patch('boto3.client') + @mock.patch("tap_dynamodb.dynamodb.Config") + def test_config_provided_request_timeout(self, mock_config, mock_client): + """ + Unit tests to ensure that request timeout is set based on config value + """ + config = {"region_name": "dummy_region", "request_timeout": 100} + dynamodb.get_client(config) + mock_config.assert_called_with(connect_timeout=100, read_timeout=100) + + @mock.patch('boto3.client') + @mock.patch("tap_dynamodb.dynamodb.Config") + def test_default_value_request_timeout(self, mock_config, mock_client): + """ + Unit tests to ensure that request timeout is set based default value + """ + config = {"region_name": "dummy_region"} + dynamodb.get_client(config) + mock_config.assert_called_with(connect_timeout=300, read_timeout=300) + + @mock.patch('boto3.client') + @mock.patch("tap_dynamodb.dynamodb.Config") + def test_config_provided_empty_request_timeout(self, mock_config, mock_client): + """ + Unit tests to ensure that request timeout is set based on default value if empty value is given in config + """ + config = {"region_name": "dummy_region", "request_timeout": ""} + dynamodb.get_client(config) + mock_config.assert_called_with(connect_timeout=300, read_timeout=300) + + @mock.patch('boto3.client') + @mock.patch("tap_dynamodb.dynamodb.Config") + def test_config_provided_string_request_timeout(self, mock_config, mock_client): + """ + Unit tests to ensure that request timeout is set based on config string value + """ + config = {"region_name": "dummy_region", "request_timeout": "100"} + dynamodb.get_client(config) + mock_config.assert_called_with(connect_timeout=100, read_timeout=100) + + @mock.patch('boto3.client') + @mock.patch("tap_dynamodb.dynamodb.Config") + def test_config_provided_float_request_timeout(self, mock_config, mock_client): + """ + Unit tests to ensure that request timeout is set based on config float value + """ + config = {"region_name": "dummy_region", "request_timeout": 100.8} + dynamodb.get_client(config) + mock_config.assert_called_with(connect_timeout=100.8, read_timeout=100.8) + + @mock.patch('boto3.client') + @mock.patch("tap_dynamodb.dynamodb.Config") + def test_stream_config_provided_request_timeout(self, mock_config, mock_client): + """ + Unit tests to ensure that request timeout is set based on config value + """ + config = {"region_name": "dummy_region", "request_timeout": 100} + dynamodb.get_stream_client(config) + mock_config.assert_called_with(connect_timeout=100, read_timeout=100) + + @mock.patch('boto3.client') + @mock.patch("tap_dynamodb.dynamodb.Config") + def test_stream_default_value_request_timeout(self, mock_config, mock_client): + """ + Unit tests to ensure that request timeout is set based default value + """ + config = {"region_name": "dummy_region"} + dynamodb.get_stream_client(config) + mock_config.assert_called_with(connect_timeout=300, read_timeout=300) + + @mock.patch('boto3.client') + @mock.patch("tap_dynamodb.dynamodb.Config") + def test_stream_config_provided_empty_request_timeout(self, mock_config, mock_client): + """ + Unit tests to ensure that request timeout is set based on default value if empty value is given in config + """ + config = {"region_name": "dummy_region", "request_timeout": ""} + dynamodb.get_stream_client(config) + mock_config.assert_called_with(connect_timeout=300, read_timeout=300) + + @mock.patch('boto3.client') + @mock.patch("tap_dynamodb.dynamodb.Config") + def test_stream_config_provided_string_request_timeout(self, mock_config, mock_client): + """ + Unit tests to ensure that request timeout is set based on config string value + """ + config = {"region_name": "dummy_region", "request_timeout": "100"} + dynamodb.get_stream_client(config) + mock_config.assert_called_with(connect_timeout=100, read_timeout=100) + + @mock.patch('boto3.client') + @mock.patch("tap_dynamodb.dynamodb.Config") + def test_stream_config_provided_float_request_timeout(self, mock_config, mock_client): + """ + Unit tests to ensure that request timeout is set based on config float value + """ + config = {"region_name": "dummy_region", "request_timeout": 100.8} + dynamodb.get_stream_client(config) + mock_config.assert_called_with(connect_timeout=100.8, read_timeout=100.8) \ No newline at end of file From 1392579d68f0ac463035d9fbcb862dee197b57f1 Mon Sep 17 00:00:00 2001 From: namrata270998 Date: Tue, 16 Nov 2021 13:04:03 +0000 Subject: [PATCH 02/15] added test cases for the backoff --- .circleci/config.yml | 11 +++ tap_dynamodb/dynamodb.py | 12 +-- tap_dynamodb/sync_strategies/full_table.py | 11 +-- tests/unittests/test_request_timeout.py | 96 ++++++++++++++++++---- 4 files changed, 105 insertions(+), 25 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8d6fa14..d2b2aae 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,6 +19,17 @@ jobs: command: | source /usr/local/share/virtualenvs/tap-dynamodb/bin/activate make lint + - run: + name: 'Unit Tests' + command: | + source /usr/local/share/virtualenvs/tap-dynamodb/bin/activate + pip install nose coverage + nosetests --with-coverage --cover-erase --cover-package=tap_dynamodb --cover-html-dir=htmlcov tests/unittests + coverage html + - store_test_results: + path: test_output/report.xml + - store_artifacts: + path: htmlcov - run: name: 'Tap Tester' command: | diff --git a/tap_dynamodb/dynamodb.py b/tap_dynamodb/dynamodb.py index 5944024..02c1e13 100644 --- a/tap_dynamodb/dynamodb.py +++ b/tap_dynamodb/dynamodb.py @@ -74,15 +74,16 @@ def get_client(config): request_timeout = float(request_timeout) else: # If value is 0,"0" or "" then set default to 300 seconds. request_timeout = REQUEST_TIMEOUT - timeout_config = Config(connect_timeout=request_timeout, read_timeout=request_timeout) + # add the request_timeout in both connect_timeout as well as read_timeout + timeout_config = Config(connect_timeout=request_timeout, read_timeout=request_timeout) if config.get('use_local_dynamo'): return boto3.client('dynamodb', endpoint_url='http://localhost:8000', region_name=config['region_name'], - config=timeout_config + config=timeout_config # pass the config to add the request_timeout ) return boto3.client('dynamodb', config['region_name'], - config=timeout_config + config=timeout_config # pass the config to add the request_timeout ) def get_stream_client(config): @@ -92,14 +93,15 @@ def get_stream_client(config): request_timeout = float(request_timeout) else: # If value is 0,"0" or "" then set default to 300 seconds. request_timeout = REQUEST_TIMEOUT + # add the request_timeout in both connect_timeout as well as read_timeout timeout_config = Config(connect_timeout=request_timeout, read_timeout=request_timeout) if config.get('use_local_dynamo'): return boto3.client('dynamodbstreams', endpoint_url='http://localhost:8000', region_name=config['region_name'], - config=timeout_config + config=timeout_config # pass the config to add the request_timeout ) return boto3.client('dynamodbstreams', region_name=config['region_name'], - config=timeout_config + config=timeout_config # pass the config to add the request_timeout ) diff --git a/tap_dynamodb/sync_strategies/full_table.py b/tap_dynamodb/sync_strategies/full_table.py index e0c9fd1..09362aa 100644 --- a/tap_dynamodb/sync_strategies/full_table.py +++ b/tap_dynamodb/sync_strategies/full_table.py @@ -9,10 +9,7 @@ LOGGER = singer.get_logger() -@backoff.on_exception(backoff.expo, - (ReadTimeoutError, ConnectTimeoutError), - max_tries=5, - factor=2) + def scan_table(table_name, projection, last_evaluated_key, config): scan_params = { 'TableName': table_name, @@ -40,7 +37,11 @@ def scan_table(table_name, projection, last_evaluated_key, config): has_more = result.get('LastEvaluatedKey', False) - +# Backoff for both ReadTimeout and ConnectTimeout error for 5 times +@backoff.on_exception(backoff.expo, + (ReadTimeoutError, ConnectTimeoutError), + max_tries=5, + factor=2) def sync(config, state, stream): table_name = stream['tap_stream_id'] diff --git a/tests/unittests/test_request_timeout.py b/tests/unittests/test_request_timeout.py index c3d0056..628510f 100644 --- a/tests/unittests/test_request_timeout.py +++ b/tests/unittests/test_request_timeout.py @@ -1,5 +1,5 @@ from tap_dynamodb.sync_strategies import full_table, log_based -from tap_dynamodb import discover, dynamodb +from tap_dynamodb import LOGGER, discover, dynamodb import tap_dynamodb import unittest from unittest import mock @@ -8,34 +8,100 @@ from botocore.exceptions import ClientError, ConnectTimeoutError, ReadTimeoutError class MockClient(): - def __init__(self, endpoint_url=None): - pass + def __init__(self,endpoint_url=None): + self.endpoint_url="abc.com" - def list_tables(self): - pass + def list_tables(self, *args): + raise ReadTimeoutError(endpoint_url="abc.com") + def scan(self, **args): + raise ReadTimeoutError(endpoint_url="abc.com") + def describe_table(self, **args): + raise ReadTimeoutError(endpoint_url="abc.com") + +def mock_get_client(*args): + return MockClient() +class MockClientConnectTimeout(): + def __init__(self,endpoint_url=None): + self.endpoint_url="abc.com" + + def list_tables(self, *args): + raise ConnectTimeoutError(endpoint_url="abc.com") + + def scan(self, **args): + raise ConnectTimeoutError(endpoint_url="abc.com") + + def describe_table(self, **args): + raise ConnectTimeoutError(endpoint_url="abc.com") + +def mock_get_client(*args): + return MockClient() + +def mock_get_client_connect_timeout(*args): + return MockClientConnectTimeout() class TestBackoffError(unittest.TestCase): ''' Test that backoff logic works properly. ''' - # @mock.patch('tap_dynamodb.dynamodb.boto3.client', return_value=MockClient()) - @mock.patch('boto3.client') - @mock.patch('singer.metadata.to_map') - def test_discover_stream_request_timeout_and_backoff(self, mock_map, mock_client): + @mock.patch('tap_dynamodb.dynamodb.get_client',side_effect=mock_get_client) + def test_discover_stream_read_timeout_and_backoff(self, mock_client): """ - Check whether the request backoffs properly for list_tables for 5 times in case of Timeout error. + Check whether the request backoffs properly for discover_streams for 5 times in case of ReadTimeoutError error. """ - mock_client.list_tables.side_effect = ReadTimeoutError - # mock_client = Mock() - # mock_client.list_tables = Mock() + with self.assertRaises(ReadTimeoutError): + discover.discover_streams({"region_name": "dummy", "use_local_dynamo": "true"}) + self.assertEquals(mock_client.call_count, 5) + @mock.patch('tap_dynamodb.dynamodb.get_client',side_effect=mock_get_client) + def test_scan_table_sync_read_timeout_and_backoff(self, mock_client): + """ + Check whether the request backoffs properly for full_table sync for 5 times in case of ReadTimeoutError error. + """ with self.assertRaises(ReadTimeoutError): - # log_based.sync({"region_name": "us-east-2"}, {}, {"tap_stream_id": "dummy", "metadata":""}) + full_table.sync({"region_name": "dummy", "use_local_dynamo": "true"}, {}, {"tap_stream_id":"dummy_stream", "metadata": ""}) + self.assertEqual(mock_client.call_count, 5) + + @mock.patch('tap_dynamodb.dynamodb.get_client',side_effect=mock_get_client) + @mock.patch('tap_dynamodb.dynamodb.get_stream_client',side_effect=mock_get_client) + def test_get_records_sync_read_timeout_and_backoff(self, mock_stream_client, mock_client): + """ + Check whether the request backoffs properly for log_based sync for 5 times in case of ReadTimeoutError error. + """ + with self.assertRaises(ReadTimeoutError): + log_based.sync({"region_name": "dummy", "use_local_dynamo": "true"}, {}, {"tap_stream_id":"dummy_stream", "metadata": ""}) + self.assertEqual(mock_client.call_count, 5) + self.assertEqual(mock_stream_client.call_count, 5) + + @mock.patch('tap_dynamodb.dynamodb.get_client',side_effect=mock_get_client_connect_timeout) + def test_discover_stream_connect_timeout_and_backoff(self, mock_client): + """ + Check whether the request backoffs properly for discover_streams for 5 times in case of ConnectTimeoutError error. + """ + with self.assertRaises(ConnectTimeoutError): discover.discover_streams({"region_name": "dummy", "use_local_dynamo": "true"}) - self.assertEquals(mock_client.list_tables.call_count, 5) + self.assertEquals(mock_client.call_count, 5) + @mock.patch('tap_dynamodb.dynamodb.get_client',side_effect=mock_get_client_connect_timeout) + def test_scan_table_sync_connect_timeout_and_backoff(self, mock_client): + """ + Check whether the request backoffs properly for full_table sync for 5 times in case of ConnectTimeoutError error. + """ + with self.assertRaises(ConnectTimeoutError): + full_table.sync({"region_name": "dummy", "use_local_dynamo": "true"}, {}, {"tap_stream_id":"dummy_stream", "metadata": ""}) + self.assertEqual(mock_client.call_count, 5) + + @mock.patch('tap_dynamodb.dynamodb.get_client',side_effect=mock_get_client_connect_timeout) + @mock.patch('tap_dynamodb.dynamodb.get_stream_client',side_effect=mock_get_client_connect_timeout) + def test_get_records_sync_connect_timeout_and_backoff(self, mock_stream_client, mock_client): + """ + Check whether the request backoffs properly for log_based sync for 5 times in case of ConnectTimeoutError error. + """ + with self.assertRaises(ConnectTimeoutError): + log_based.sync({"region_name": "dummy", "use_local_dynamo": "true"}, {}, {"tap_stream_id":"dummy_stream", "metadata": ""}) + self.assertEqual(mock_client.call_count, 5) + self.assertEqual(mock_stream_client.call_count, 5) class TestRequestTimeoutValue(unittest.TestCase): ''' From 3a9c8fc000015f09992a7096476b7338cd211994 Mon Sep 17 00:00:00 2001 From: namrata270998 Date: Tue, 16 Nov 2021 13:12:02 +0000 Subject: [PATCH 03/15] resolved pylint errors --- tap_dynamodb/discover.py | 3 +-- tap_dynamodb/sync_strategies/full_table.py | 4 ++-- tap_dynamodb/sync_strategies/log_based.py | 3 +-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tap_dynamodb/discover.py b/tap_dynamodb/discover.py index c4b9534..6060032 100644 --- a/tap_dynamodb/discover.py +++ b/tap_dynamodb/discover.py @@ -1,9 +1,8 @@ from singer import metadata import singer -from botocore.exceptions import ClientError -from tap_dynamodb import dynamodb import backoff from botocore.exceptions import ClientError, ConnectTimeoutError, ReadTimeoutError +from tap_dynamodb import dynamodb LOGGER = singer.get_logger() diff --git a/tap_dynamodb/sync_strategies/full_table.py b/tap_dynamodb/sync_strategies/full_table.py index 09362aa..59d2971 100644 --- a/tap_dynamodb/sync_strategies/full_table.py +++ b/tap_dynamodb/sync_strategies/full_table.py @@ -2,10 +2,10 @@ import singer from singer import metadata -from tap_dynamodb import dynamodb from tap_dynamodb.deserialize import Deserializer import backoff -from botocore.exceptions import ClientError, ConnectTimeoutError, ReadTimeoutError +from botocore.exceptions import ConnectTimeoutError, ReadTimeoutError +from tap_dynamodb import dynamodb LOGGER = singer.get_logger() diff --git a/tap_dynamodb/sync_strategies/log_based.py b/tap_dynamodb/sync_strategies/log_based.py index a05fc8f..8c1bed9 100644 --- a/tap_dynamodb/sync_strategies/log_based.py +++ b/tap_dynamodb/sync_strategies/log_based.py @@ -1,11 +1,10 @@ import datetime from singer import metadata import singer - -from tap_dynamodb import dynamodb from tap_dynamodb import deserialize import backoff from botocore.exceptions import ConnectTimeoutError, ReadTimeoutError +from tap_dynamodb import dynamodb LOGGER = singer.get_logger() WRITE_STATE_PERIOD = 1000 From e658b3d2dc117144a1c2dd6ab2f6f590b2a670fa Mon Sep 17 00:00:00 2001 From: namrata270998 Date: Tue, 16 Nov 2021 13:14:33 +0000 Subject: [PATCH 04/15] updated pylint --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d2b2aae..fad39d0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,7 @@ jobs: name: 'pylint' command: | source /usr/local/share/virtualenvs/tap-dynamodb/bin/activate - make lint + pylint tap_dynamodb --disable 'duplicate-code' - run: name: 'Unit Tests' command: | From f7977e2cf2257fe382f381e06c882506296b00b0 Mon Sep 17 00:00:00 2001 From: namrata270998 Date: Tue, 16 Nov 2021 13:20:17 +0000 Subject: [PATCH 05/15] fixd pylint --- .circleci/config.yml | 2 +- tap_dynamodb/sync_strategies/log_based.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fad39d0..e4cc9f5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,7 @@ jobs: name: 'pylint' command: | source /usr/local/share/virtualenvs/tap-dynamodb/bin/activate - pylint tap_dynamodb --disable 'duplicate-code' + pylint tap_dynamodb --disable 'duplicate-code,missing-module-docstring,line-too-long,fixme,missing-function-docstring,raise-missing-from,consider-using-f-string,too-many-locals,invalid-name' - run: name: 'Unit Tests' command: | diff --git a/tap_dynamodb/sync_strategies/log_based.py b/tap_dynamodb/sync_strategies/log_based.py index 8c1bed9..adf420c 100644 --- a/tap_dynamodb/sync_strategies/log_based.py +++ b/tap_dynamodb/sync_strategies/log_based.py @@ -1,10 +1,9 @@ import datetime from singer import metadata import singer -from tap_dynamodb import deserialize import backoff from botocore.exceptions import ConnectTimeoutError, ReadTimeoutError -from tap_dynamodb import dynamodb +from tap_dynamodb import dynamodb, deserialize LOGGER = singer.get_logger() WRITE_STATE_PERIOD = 1000 From 4ab905a18bfd8471c16a84404578f125b430d05d Mon Sep 17 00:00:00 2001 From: namrata270998 Date: Tue, 16 Nov 2021 13:54:07 +0000 Subject: [PATCH 06/15] updated config.yml --- .circleci/config.yml | 2 +- tap_dynamodb/sync_strategies/full_table.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e4cc9f5..e12a834 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,7 @@ jobs: name: 'pylint' command: | source /usr/local/share/virtualenvs/tap-dynamodb/bin/activate - pylint tap_dynamodb --disable 'duplicate-code,missing-module-docstring,line-too-long,fixme,missing-function-docstring,raise-missing-from,consider-using-f-string,too-many-locals,invalid-name' + pylint tap_dynamodb --disable 'duplicate-code,missing-module-docstring,missing-class-docstring,too-few-public-methods,line-too-long,fixme,missing-function-docstring,raise-missing-from,consider-using-f-string,too-many-locals,invalid-name,too-many-arguments' - run: name: 'Unit Tests' command: | diff --git a/tap_dynamodb/sync_strategies/full_table.py b/tap_dynamodb/sync_strategies/full_table.py index 59d2971..9eecb83 100644 --- a/tap_dynamodb/sync_strategies/full_table.py +++ b/tap_dynamodb/sync_strategies/full_table.py @@ -2,9 +2,9 @@ import singer from singer import metadata -from tap_dynamodb.deserialize import Deserializer import backoff from botocore.exceptions import ConnectTimeoutError, ReadTimeoutError +from tap_dynamodb.deserialize import Deserializer from tap_dynamodb import dynamodb LOGGER = singer.get_logger() From 2d6145eb0032d45db953ac5ff0083c14942c2c30 Mon Sep 17 00:00:00 2001 From: namrata270998 Date: Fri, 26 Nov 2021 12:26:08 +0000 Subject: [PATCH 07/15] resolved comments --- .circleci/config.yml | 2 +- tap_dynamodb/dynamodb.py | 15 ++++++++------- tap_dynamodb/sync_strategies/full_table.py | 6 +++--- tap_dynamodb/sync_strategies/log_based.py | 6 +++--- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e12a834..d2b2aae 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,7 @@ jobs: name: 'pylint' command: | source /usr/local/share/virtualenvs/tap-dynamodb/bin/activate - pylint tap_dynamodb --disable 'duplicate-code,missing-module-docstring,missing-class-docstring,too-few-public-methods,line-too-long,fixme,missing-function-docstring,raise-missing-from,consider-using-f-string,too-many-locals,invalid-name,too-many-arguments' + make lint - run: name: 'Unit Tests' command: | diff --git a/tap_dynamodb/dynamodb.py b/tap_dynamodb/dynamodb.py index 02c1e13..0d90597 100644 --- a/tap_dynamodb/dynamodb.py +++ b/tap_dynamodb/dynamodb.py @@ -67,13 +67,18 @@ def setup_aws_client(config): LOGGER.info("Attempting to assume_role on RoleArn: %s", role_arn) boto3.setup_default_session(botocore_session=refreshable_session) -def get_client(config): +def get_request_timeout(config): # if request_timeout is other than 0,"0" or "" then use request_timeout request_timeout = config.get('request_timeout') if request_timeout and float(request_timeout): request_timeout = float(request_timeout) else: # If value is 0,"0" or "" then set default to 300 seconds. request_timeout = REQUEST_TIMEOUT + return request_timeout + +def get_client(config): + # get the request_timeout + request_timeout = get_request_timeout(config) # add the request_timeout in both connect_timeout as well as read_timeout timeout_config = Config(connect_timeout=request_timeout, read_timeout=request_timeout) if config.get('use_local_dynamo'): @@ -87,12 +92,8 @@ def get_client(config): ) def get_stream_client(config): - # if request_timeout is other than 0,"0" or "" then use request_timeout - request_timeout = config.get('request_timeout') - if request_timeout and float(request_timeout): - request_timeout = float(request_timeout) - else: # If value is 0,"0" or "" then set default to 300 seconds. - request_timeout = REQUEST_TIMEOUT + # get the request_timeout + request_timeout = get_request_timeout(config) # add the request_timeout in both connect_timeout as well as read_timeout timeout_config = Config(connect_timeout=request_timeout, read_timeout=request_timeout) if config.get('use_local_dynamo'): diff --git a/tap_dynamodb/sync_strategies/full_table.py b/tap_dynamodb/sync_strategies/full_table.py index 9eecb83..51e90d5 100644 --- a/tap_dynamodb/sync_strategies/full_table.py +++ b/tap_dynamodb/sync_strategies/full_table.py @@ -39,9 +39,9 @@ def scan_table(table_name, projection, last_evaluated_key, config): # Backoff for both ReadTimeout and ConnectTimeout error for 5 times @backoff.on_exception(backoff.expo, - (ReadTimeoutError, ConnectTimeoutError), - max_tries=5, - factor=2) + (ReadTimeoutError, ConnectTimeoutError), + max_tries=5, + factor=2) def sync(config, state, stream): table_name = stream['tap_stream_id'] diff --git a/tap_dynamodb/sync_strategies/log_based.py b/tap_dynamodb/sync_strategies/log_based.py index adf420c..8fc98c3 100644 --- a/tap_dynamodb/sync_strategies/log_based.py +++ b/tap_dynamodb/sync_strategies/log_based.py @@ -117,9 +117,9 @@ def sync_shard(shard, seq_number_bookmarks, streams_client, stream_arn, projecti # Backoff for both ReadTimeout and ConnectTimeout error for 5 times @backoff.on_exception(backoff.expo, - (ReadTimeoutError, ConnectTimeoutError), - max_tries=5, - factor=2) + (ReadTimeoutError, ConnectTimeoutError), + max_tries=5, + factor=2) def sync(config, state, stream): table_name = stream['tap_stream_id'] From b301a0621816dfa6666ec9f78409ae7e3022501c Mon Sep 17 00:00:00 2001 From: namrata270998 Date: Fri, 26 Nov 2021 12:31:38 +0000 Subject: [PATCH 08/15] resolved comments --- tap_dynamodb/sync_strategies/log_based.py | 7 ++++--- tests/unittests/test_request_timeout.py | 9 +++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/tap_dynamodb/sync_strategies/log_based.py b/tap_dynamodb/sync_strategies/log_based.py index 8fc98c3..6cd05a2 100644 --- a/tap_dynamodb/sync_strategies/log_based.py +++ b/tap_dynamodb/sync_strategies/log_based.py @@ -216,10 +216,11 @@ def has_stream_aged_out(state, table_name): # stream then we consider the stream to be aged out return time_span > datetime.timedelta(hours=19, minutes=30) +# Backoff for both ReadTimeout and ConnectTimeout error for 5 times @backoff.on_exception(backoff.expo, - (ReadTimeoutError, ConnectTimeoutError), - max_tries=5, - factor=2) + (ReadTimeoutError, ConnectTimeoutError), + max_tries=5, + factor=2) def get_initial_bookmarks(config, state, table_name): ''' Returns the state including all bookmarks necessary for the initial diff --git a/tests/unittests/test_request_timeout.py b/tests/unittests/test_request_timeout.py index 628510f..439f689 100644 --- a/tests/unittests/test_request_timeout.py +++ b/tests/unittests/test_request_timeout.py @@ -103,6 +103,15 @@ def test_get_records_sync_connect_timeout_and_backoff(self, mock_stream_client, self.assertEqual(mock_client.call_count, 5) self.assertEqual(mock_stream_client.call_count, 5) + @mock.patch('tap_dynamodb.dynamodb.get_client',side_effect=mock_get_client) + def test_get_initial_bookmarks_read_timeout_and_backoff(self, mock_client): + """ + Check whether the request backoffs properly for discover_streams for 5 times in case of ReadTimeoutError error. + """ + with self.assertRaises(ReadTimeoutError): + log_based.get_initial_bookmarks({"region_name": "dummy", "use_local_dynamo": "true"}, {}, "dummy_table") + self.assertEquals(mock_client.call_count, 5) + class TestRequestTimeoutValue(unittest.TestCase): ''' Test that request timeout parameter works properly in various cases From 68439cf348f7b5748b8e6f66f379d35386411763 Mon Sep 17 00:00:00 2001 From: namrata270998 Date: Fri, 26 Nov 2021 12:33:17 +0000 Subject: [PATCH 09/15] resolved pylint errors --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 791b466..1eca0b0 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,6 @@ lint-tests: pylint tests -d broad-except,chained-comparison,empty-docstring,fixme,invalid-name,line-too-long,missing-class-docstring,missing-function-docstring,missing-module-docstring,no-else-raise,no-else-return,too-few-public-methods,too-many-arguments,too-many-branches,too-many-lines,too-many-locals,ungrouped-imports,wrong-spelling-in-comment,wrong-spelling-in-docstring,duplicate-code,no-name-in-module,import-error,consider-using-f-string lint-code: - pylint tap_dynamodb -d broad-except,chained-comparison,empty-docstring,fixme,invalid-name,line-too-long,missing-class-docstring,missing-function-docstring,missing-module-docstring,no-else-raise,no-else-return,too-few-public-methods,too-many-arguments,too-many-branches,too-many-lines,too-many-locals,ungrouped-imports,wrong-spelling-in-comment,wrong-spelling-in-docstring,raise-missing-from,consider-using-f-string + pylint tap_dynamodb -d broad-except,chained-comparison,empty-docstring,fixme,invalid-name,line-too-long,missing-class-docstring,missing-function-docstring,missing-module-docstring,no-else-raise,no-else-return,too-few-public-methods,too-many-arguments,too-many-branches,too-many-lines,too-many-locals,ungrouped-imports,wrong-spelling-in-comment,wrong-spelling-in-docstring,raise-missing-from,consider-using-f-string,duplicate-code lint: lint-code lint-tests From 30e21ad9cd05534299276b7d8ef6b5e6df5cbd1d Mon Sep 17 00:00:00 2001 From: namrata270998 Date: Fri, 7 Jan 2022 15:30:27 +0530 Subject: [PATCH 10/15] resolved comments --- Makefile | 2 +- tap_dynamodb/sync_strategies/log_based.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 1eca0b0..791b466 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,6 @@ lint-tests: pylint tests -d broad-except,chained-comparison,empty-docstring,fixme,invalid-name,line-too-long,missing-class-docstring,missing-function-docstring,missing-module-docstring,no-else-raise,no-else-return,too-few-public-methods,too-many-arguments,too-many-branches,too-many-lines,too-many-locals,ungrouped-imports,wrong-spelling-in-comment,wrong-spelling-in-docstring,duplicate-code,no-name-in-module,import-error,consider-using-f-string lint-code: - pylint tap_dynamodb -d broad-except,chained-comparison,empty-docstring,fixme,invalid-name,line-too-long,missing-class-docstring,missing-function-docstring,missing-module-docstring,no-else-raise,no-else-return,too-few-public-methods,too-many-arguments,too-many-branches,too-many-lines,too-many-locals,ungrouped-imports,wrong-spelling-in-comment,wrong-spelling-in-docstring,raise-missing-from,consider-using-f-string,duplicate-code + pylint tap_dynamodb -d broad-except,chained-comparison,empty-docstring,fixme,invalid-name,line-too-long,missing-class-docstring,missing-function-docstring,missing-module-docstring,no-else-raise,no-else-return,too-few-public-methods,too-many-arguments,too-many-branches,too-many-lines,too-many-locals,ungrouped-imports,wrong-spelling-in-comment,wrong-spelling-in-docstring,raise-missing-from,consider-using-f-string lint: lint-code lint-tests diff --git a/tap_dynamodb/sync_strategies/log_based.py b/tap_dynamodb/sync_strategies/log_based.py index 6cd05a2..c7903a7 100644 --- a/tap_dynamodb/sync_strategies/log_based.py +++ b/tap_dynamodb/sync_strategies/log_based.py @@ -9,7 +9,8 @@ WRITE_STATE_PERIOD = 1000 SDC_DELETED_AT = "_sdc_deleted_at" - +MAX_TRIES = 5 +FACTOR = 2 def get_shards(streams_client, stream_arn): ''' @@ -118,8 +119,8 @@ def sync_shard(shard, seq_number_bookmarks, streams_client, stream_arn, projecti # Backoff for both ReadTimeout and ConnectTimeout error for 5 times @backoff.on_exception(backoff.expo, (ReadTimeoutError, ConnectTimeoutError), - max_tries=5, - factor=2) + max_tries=MAX_TRIES, + factor=FACTOR) def sync(config, state, stream): table_name = stream['tap_stream_id'] @@ -219,8 +220,8 @@ def has_stream_aged_out(state, table_name): # Backoff for both ReadTimeout and ConnectTimeout error for 5 times @backoff.on_exception(backoff.expo, (ReadTimeoutError, ConnectTimeoutError), - max_tries=5, - factor=2) + max_tries=MAX_TRIES, + factor=FACTOR) def get_initial_bookmarks(config, state, table_name): ''' Returns the state including all bookmarks necessary for the initial From def47709ec3ed4bb2518415d13f85157985266e3 Mon Sep 17 00:00:00 2001 From: namrata270998 Date: Fri, 7 Jan 2022 15:45:24 +0530 Subject: [PATCH 11/15] resolved comments --- tests/unittests/test_request_timeout.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/unittests/test_request_timeout.py b/tests/unittests/test_request_timeout.py index 439f689..d95b815 100644 --- a/tests/unittests/test_request_timeout.py +++ b/tests/unittests/test_request_timeout.py @@ -1,11 +1,9 @@ from tap_dynamodb.sync_strategies import full_table, log_based -from tap_dynamodb import LOGGER, discover, dynamodb +from tap_dynamodb import discover, dynamodb import tap_dynamodb import unittest from unittest import mock -from unittest.mock import Mock -from unittest.case import TestCase -from botocore.exceptions import ClientError, ConnectTimeoutError, ReadTimeoutError +from botocore.exceptions import ConnectTimeoutError, ReadTimeoutError class MockClient(): def __init__(self,endpoint_url=None): From 45b9b8035c3185eaab4d6ee9f4f0dd8824a540b9 Mon Sep 17 00:00:00 2001 From: namrata270998 Date: Thu, 13 Jan 2022 14:56:29 +0530 Subject: [PATCH 12/15] resolved comments --- .circleci/config.yml | 12 ++++------ tests/unittests/test_request_timeout.py | 29 +++++++++++++++---------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d2b2aae..34fc42b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2 jobs: build: docker: - - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:tap-tester-v4 + - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:stitch-tap-tester - image: amazon/dynamodb-local entrypoint: ["java", "-Xmx1G", "-jar", "DynamoDBLocal.jar"] steps: @@ -36,13 +36,9 @@ jobs: aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/sandbox tap-tester.env source tap-tester.env source /usr/local/share/virtualenvs/tap-tester/bin/activate - run-test --tap=tap-dynamodb \ - --target=target-stitch \ - --orchestrator=stitch-orchestrator \ - --email=harrison+sandboxtest@stitchdata.com \ - --password=$SANDBOX_PASSWORD \ - --client-id=50 \ - tests + run-test --tap=tap-dynamodb tests + - slack/notify-on-failure: + only_for_branches: master workflows: version: 2 commit: diff --git a/tests/unittests/test_request_timeout.py b/tests/unittests/test_request_timeout.py index d95b815..0b08174 100644 --- a/tests/unittests/test_request_timeout.py +++ b/tests/unittests/test_request_timeout.py @@ -114,15 +114,17 @@ class TestRequestTimeoutValue(unittest.TestCase): ''' Test that request timeout parameter works properly in various cases ''' + default_timeout_value = 300 @mock.patch('boto3.client') @mock.patch("tap_dynamodb.dynamodb.Config") def test_config_provided_request_timeout(self, mock_config, mock_client): """ Unit tests to ensure that request timeout is set based on config value """ - config = {"region_name": "dummy_region", "request_timeout": 100} + timeout_value = 100 + config = {"region_name": "dummy_region", "request_timeout": timeout_value} dynamodb.get_client(config) - mock_config.assert_called_with(connect_timeout=100, read_timeout=100) + mock_config.assert_called_with(connect_timeout=timeout_value, read_timeout=timeout_value) @mock.patch('boto3.client') @mock.patch("tap_dynamodb.dynamodb.Config") @@ -132,7 +134,7 @@ def test_default_value_request_timeout(self, mock_config, mock_client): """ config = {"region_name": "dummy_region"} dynamodb.get_client(config) - mock_config.assert_called_with(connect_timeout=300, read_timeout=300) + mock_config.assert_called_with(connect_timeout=self.default_timeout_value, read_timeout=self.default_timeout_value) @mock.patch('boto3.client') @mock.patch("tap_dynamodb.dynamodb.Config") @@ -142,7 +144,7 @@ def test_config_provided_empty_request_timeout(self, mock_config, mock_client): """ config = {"region_name": "dummy_region", "request_timeout": ""} dynamodb.get_client(config) - mock_config.assert_called_with(connect_timeout=300, read_timeout=300) + mock_config.assert_called_with(connect_timeout=self.default_timeout_value, read_timeout=self.default_timeout_value) @mock.patch('boto3.client') @mock.patch("tap_dynamodb.dynamodb.Config") @@ -160,9 +162,10 @@ def test_config_provided_float_request_timeout(self, mock_config, mock_client): """ Unit tests to ensure that request timeout is set based on config float value """ - config = {"region_name": "dummy_region", "request_timeout": 100.8} + timeout_value = 100.8 + config = {"region_name": "dummy_region", "request_timeout": timeout_value} dynamodb.get_client(config) - mock_config.assert_called_with(connect_timeout=100.8, read_timeout=100.8) + mock_config.assert_called_with(connect_timeout=timeout_value, read_timeout=timeout_value) @mock.patch('boto3.client') @mock.patch("tap_dynamodb.dynamodb.Config") @@ -170,9 +173,10 @@ def test_stream_config_provided_request_timeout(self, mock_config, mock_client): """ Unit tests to ensure that request timeout is set based on config value """ - config = {"region_name": "dummy_region", "request_timeout": 100} + timeout_value = 100 + config = {"region_name": "dummy_region", "request_timeout": timeout_value} dynamodb.get_stream_client(config) - mock_config.assert_called_with(connect_timeout=100, read_timeout=100) + mock_config.assert_called_with(connect_timeout=timeout_value, read_timeout=timeout_value) @mock.patch('boto3.client') @mock.patch("tap_dynamodb.dynamodb.Config") @@ -182,7 +186,7 @@ def test_stream_default_value_request_timeout(self, mock_config, mock_client): """ config = {"region_name": "dummy_region"} dynamodb.get_stream_client(config) - mock_config.assert_called_with(connect_timeout=300, read_timeout=300) + mock_config.assert_called_with(connect_timeout=self.default_timeout_value, read_timeout=self.default_timeout_value) @mock.patch('boto3.client') @mock.patch("tap_dynamodb.dynamodb.Config") @@ -192,7 +196,7 @@ def test_stream_config_provided_empty_request_timeout(self, mock_config, mock_cl """ config = {"region_name": "dummy_region", "request_timeout": ""} dynamodb.get_stream_client(config) - mock_config.assert_called_with(connect_timeout=300, read_timeout=300) + mock_config.assert_called_with(connect_timeout=self.default_timeout_value, read_timeout=self.default_timeout_value) @mock.patch('boto3.client') @mock.patch("tap_dynamodb.dynamodb.Config") @@ -210,6 +214,7 @@ def test_stream_config_provided_float_request_timeout(self, mock_config, mock_cl """ Unit tests to ensure that request timeout is set based on config float value """ - config = {"region_name": "dummy_region", "request_timeout": 100.8} + timeout_value = 100.8 + config = {"region_name": "dummy_region", "request_timeout": timeout_value} dynamodb.get_stream_client(config) - mock_config.assert_called_with(connect_timeout=100.8, read_timeout=100.8) \ No newline at end of file + mock_config.assert_called_with(connect_timeout=timeout_value, read_timeout=timeout_value) \ No newline at end of file From 3f42e0fd7f6f1fd93be1b699052fa22429f0afe9 Mon Sep 17 00:00:00 2001 From: namrata270998 Date: Thu, 13 Jan 2022 15:05:08 +0530 Subject: [PATCH 13/15] resolved slack-on-notify failure --- .circleci/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 34fc42b..4f0d190 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,4 +1,7 @@ -version: 2 +version: 2.1 +orbs: + slack: circleci/slack@3.4.2 + jobs: build: docker: From 764bae61e06f86a5aec3b456a2367f87bcc64530 Mon Sep 17 00:00:00 2001 From: namrata270998 Date: Thu, 13 Jan 2022 15:16:08 +0530 Subject: [PATCH 14/15] removed scenarios --- tests/test_dynamodb_discovery.py | 4 ---- tests/test_dynamodb_full_table_interruptible_sync.py | 4 ---- tests/test_dynamodb_full_table_sync.py | 3 --- tests/test_dynamodb_log_based.py | 4 ---- tests/test_dynamodb_log_based_interruptible.py | 5 ----- tests/test_dynamodb_log_based_projections.py | 4 ---- tests/test_dynamodb_projections.py | 4 ---- 7 files changed, 28 deletions(-) diff --git a/tests/test_dynamodb_discovery.py b/tests/test_dynamodb_discovery.py index d20df97..d5da72a 100644 --- a/tests/test_dynamodb_discovery.py +++ b/tests/test_dynamodb_discovery.py @@ -1,6 +1,5 @@ from boto3.dynamodb.types import TypeSerializer -from tap_tester.scenario import (SCENARIOS) from base import TestDynamoDBBase @@ -53,6 +52,3 @@ def name(): def test_run(self): self.pre_sync_test() - - -SCENARIOS.add(DynamoDBDiscovery) diff --git a/tests/test_dynamodb_full_table_interruptible_sync.py b/tests/test_dynamodb_full_table_interruptible_sync.py index 24902a5..c16c782 100644 --- a/tests/test_dynamodb_full_table_interruptible_sync.py +++ b/tests/test_dynamodb_full_table_interruptible_sync.py @@ -3,7 +3,6 @@ from boto3.dynamodb.types import TypeSerializer -from tap_tester.scenario import (SCENARIOS) from tap_tester import connections from tap_tester import menagerie from tap_tester import runner @@ -129,6 +128,3 @@ def test_run(self): self.assertIsNone(state['bookmarks'][table_name].get('last_evaluated_key')) self.assertTrue(state['bookmarks'][table_name].get('initial_full_table_complete', False)) - - -SCENARIOS.add(DynamoDBFullTableInterruptible) diff --git a/tests/test_dynamodb_full_table_sync.py b/tests/test_dynamodb_full_table_sync.py index 77f055e..e4135cc 100644 --- a/tests/test_dynamodb_full_table_sync.py +++ b/tests/test_dynamodb_full_table_sync.py @@ -4,7 +4,6 @@ from boto3.dynamodb.types import TypeSerializer -from tap_tester.scenario import (SCENARIOS) from tap_tester import connections from tap_tester import menagerie from tap_tester import runner @@ -108,5 +107,3 @@ def test_run(self): # assert that there is a version bookmark in state first_versions[table_name] = state['bookmarks'][table_name]['version'] self.assertIsNotNone(first_versions[table_name]) - -SCENARIOS.add(DynamoDBFullTable) diff --git a/tests/test_dynamodb_log_based.py b/tests/test_dynamodb_log_based.py index 79d14ea..7bc39f4 100644 --- a/tests/test_dynamodb_log_based.py +++ b/tests/test_dynamodb_log_based.py @@ -2,7 +2,6 @@ from boto3.dynamodb.types import TypeSerializer -from tap_tester.scenario import (SCENARIOS) from tap_tester import connections from tap_tester import menagerie from tap_tester import runner @@ -143,6 +142,3 @@ def test_run(self): self.assertEqual(31, len(stream['messages'])) state = menagerie.get_state(conn_id) - - -SCENARIOS.add(DynamoDBLogBased) diff --git a/tests/test_dynamodb_log_based_interruptible.py b/tests/test_dynamodb_log_based_interruptible.py index eaf39f6..b7937a3 100644 --- a/tests/test_dynamodb_log_based_interruptible.py +++ b/tests/test_dynamodb_log_based_interruptible.py @@ -2,7 +2,6 @@ from boto3.dynamodb.types import TypeSerializer -from tap_tester.scenario import (SCENARIOS) from tap_tester import connections from tap_tester import menagerie from tap_tester import runner @@ -202,7 +201,3 @@ def first_sync_test(self, table_configs, conn_id): # as the full table sync state['bookmarks'][table_name].pop('finished_shards') menagerie.set_state(conn_id, state, version=state_version) - - - -SCENARIOS.add(DynamoDBLogBased) diff --git a/tests/test_dynamodb_log_based_projections.py b/tests/test_dynamodb_log_based_projections.py index 7c79e69..8b7998b 100644 --- a/tests/test_dynamodb_log_based_projections.py +++ b/tests/test_dynamodb_log_based_projections.py @@ -3,7 +3,6 @@ from boto3.dynamodb.types import TypeSerializer -from tap_tester.scenario import (SCENARIOS) from tap_tester import connections from tap_tester import menagerie from tap_tester import runner @@ -205,6 +204,3 @@ def first_sync_test(self, table_configs, conn_id, expected_streams): for list_key in config['top_level_list_keys']: self.assertTrue(isinstance(message['data'][list_key], list)) self.assertEqual(config['nested_map_keys']['map_field'], {*message['data']['map_field'].keys()}) - - -SCENARIOS.add(DynamoDBLogBasedProjections) diff --git a/tests/test_dynamodb_projections.py b/tests/test_dynamodb_projections.py index 0f8c416..596735b 100644 --- a/tests/test_dynamodb_projections.py +++ b/tests/test_dynamodb_projections.py @@ -3,7 +3,6 @@ from boto3.dynamodb.types import TypeSerializer -from tap_tester.scenario import (SCENARIOS) from tap_tester import connections from tap_tester import menagerie from tap_tester import runner @@ -134,6 +133,3 @@ def test_run(self): for list_key in config['top_level_list_keys']: self.assertTrue(isinstance(message['data'][list_key], list)) self.assertEqual(config['nested_map_keys']['map_field'], {*message['data']['map_field'].keys()}) - - -SCENARIOS.add(DynamoDBProjections) From 0e85aee39e856b38a6f73f262863fa93f9d55064 Mon Sep 17 00:00:00 2001 From: namrata270998 Date: Thu, 27 Jan 2022 11:36:11 +0530 Subject: [PATCH 15/15] added tap-tester-user in config --- .circleci/config.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4f0d190..2e2ee1a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,7 +36,7 @@ jobs: - run: name: 'Tap Tester' command: | - aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/sandbox tap-tester.env + aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/tap_tester_sandbox tap-tester.env source tap-tester.env source /usr/local/share/virtualenvs/tap-tester/bin/activate run-test --tap=tap-dynamodb tests @@ -47,7 +47,9 @@ workflows: commit: jobs: - build: - context: circleci-user + context: + - circleci-user + - tap-tester-user build_daily: triggers: - schedule: @@ -58,4 +60,6 @@ workflows: - master jobs: - build: - context: circleci-user + context: + - circleci-user + - tap-tester-user