From 9cd8ea3e136b4c64269e1f1ef15a4876c10adf61 Mon Sep 17 00:00:00 2001 From: Alexej Tessaro Date: Thu, 11 Aug 2016 15:06:49 +0700 Subject: [PATCH 1/2] Force plain (not gzipped) aws response on aiohttp request When using aiobotocore with dynamodb, requests fail on crc32 checksum computation as soon as the response data reaches ~5KB. It seems that this is due to aws sending data back in gzip compressed format, aiohttp automatically uncompressing it and crc32 checksum being computed on the uncompressed data bytes. While crc32 has to be computed on compressed data in order to pass. The same requests work as expected when using botocore, this commit forces aws not to use gzip compression. --- aiobotocore/endpoint.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/aiobotocore/endpoint.py b/aiobotocore/endpoint.py index 78f75531..4820aa46 100644 --- a/aiobotocore/endpoint.py +++ b/aiobotocore/endpoint.py @@ -154,6 +154,18 @@ def __init__(self, host, @asyncio.coroutine def _request(self, method, url, headers, data): + # Note: When using aiobotocore with dynamodb, requests fail on crc32 + # checksum computation as soon as the response data reaches ~5KB. + # When aws response is gzip compressed: + # 1. aiohttp is automatically uncompressessing the data + # (http://aiohttp.readthedocs.io/en/stable/client.html#binary-response-content) + # 2. botocore computes crc32 on the uncompressed data bytes and fails + # cause crc32 has been computed on the compressed data + # The following line forces aws not to use gzip compression, + # if there is a way to configure aiohttp not to perform uncompression, + # we can remove the following line and take advantage of + # aws gzip compression. + headers['Accept-Encoding'] = 'identity' headers_ = dict( (z[0], text_(z[1], encoding='utf-8')) for z in headers.items()) request_coro = self._aio_session.request(method, url=url, From 0dae162a155232b8acd967e2476c74428add4104 Mon Sep 17 00:00:00 2001 From: Alexej Tessaro Date: Fri, 14 Oct 2016 17:54:34 +0700 Subject: [PATCH 2/2] Add basic dynamodb test --- tests/conftest.py | 113 +++++++++++++++++++++++++++++++++-- tests/test_basic_dynamodb.py | 19 ++++++ 2 files changed, 126 insertions(+), 6 deletions(-) create mode 100644 tests/test_basic_dynamodb.py diff --git a/tests/conftest.py b/tests/conftest.py index e7aa98be..f7190a6a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -58,9 +58,17 @@ def pytest_runtest_setup(item): def random_bucketname(): # 63 is the max bucket length. - bucket_name = 'aiobotocoretest_{}' + return random_name() + + +def random_tablename(): + return random_name() + + +def random_name(): + table_name = 'aiobotocoretest_{}' t = time.time() - return bucket_name.format(int(t)) + return table_name.format(int(t)) def assert_status_code(response, status_code): @@ -114,14 +122,25 @@ def signature_version(): @pytest.fixture -def config(signature_version, region): - conf = AioConfig(region_name=region, signature_version=signature_version) - return conf +def config(region, signature_version): + return AioConfig(region_name=region, signature_version=signature_version) @pytest.fixture def s3_client(request, session, region, config): - client = session.create_client('s3', region_name=region, config=config) + client = create_client('s3', request, session, region, config) + return client + + +@pytest.fixture +def dynamodb_client(request, session, region, config): + client = create_client('dynamodb', request, session, region, config) + return client + + +def create_client(client_type, request, session, region, config): + client = session.create_client(client_type, region_name=region, + config=config) def fin(): client.close() @@ -159,6 +178,12 @@ def bucket_name(region, create_bucket, s3_client, loop): return name +@pytest.fixture +def table_name(region, create_table, dynamodb_client, loop): + name = loop.run_until_complete(create_table()) + return name + + @pytest.fixture def create_bucket(request, s3_client, loop): _bucket_name = None @@ -186,6 +211,64 @@ def fin(): return _f +@pytest.fixture +def create_table(request, dynamodb_client, loop): + _table_name = None + + @asyncio.coroutine + def _is_table_ready(table_name): + response = yield from dynamodb_client.describe_table( + TableName=table_name + ) + return response['Table']['TableStatus'] == 'ACTIVE' + + @asyncio.coroutine + def _f(table_name=None): + + nonlocal _table_name + if table_name is None: + table_name = random_tablename() + _table_name = table_name + table_kwargs = { + 'TableName': table_name, + 'AttributeDefinitions': [ + { + 'AttributeName': 'testKey', + 'AttributeType': 'S' + }, + ], + 'KeySchema': [ + { + 'AttributeName': 'testKey', + 'KeyType': 'HASH' + }, + ], + 'ProvisionedThroughput': { + 'ReadCapacityUnits': 1, + 'WriteCapacityUnits': 1 + }, + } + response = yield from dynamodb_client.create_table(**table_kwargs) + while not (yield from _is_table_ready(table_name)): + pass + assert_status_code(response, 200) + return table_name + + def fin(): + loop.run_until_complete(delete_table(dynamodb_client, _table_name)) + + request.addfinalizer(fin) + return _f + + +@asyncio.coroutine +def delete_table(dynamodb_client, table_name): + response = yield from dynamodb_client.delete_table( + TableName=table_name + ) + assert_status_code(response, 200) + + @pytest.fixture def tempdir(request): tempdir = tempfile.mkdtemp() @@ -247,3 +330,21 @@ def pytest_namespace(): return {'aio': {'assert_status_code': assert_status_code, 'assert_num_uploads_found': assert_num_uploads_found}, } + + +@pytest.fixture +def dynamodb_put_item(request, dynamodb_client, table_name, loop): + + @asyncio.coroutine + def _f(key_string_value): + response = yield from dynamodb_client.put_item( + TableName=table_name, + Item={ + 'testKey': { + 'S': key_string_value + } + }, + ) + assert_status_code(response, 200) + + return _f diff --git a/tests/test_basic_dynamodb.py b/tests/test_basic_dynamodb.py new file mode 100644 index 00000000..225e8bd4 --- /dev/null +++ b/tests/test_basic_dynamodb.py @@ -0,0 +1,19 @@ +# import asyncio +import pytest + + +@pytest.mark.parametrize('signature_version', ['v4']) +@pytest.mark.run_loop +def test_can_get_item(dynamodb_client, table_name, dynamodb_put_item): + test_value = 'testValue' + yield from dynamodb_put_item(test_value) + response = yield from dynamodb_client.get_item( + TableName=table_name, + Key={ + 'testKey': { + 'S': test_value + } + }, + ) + pytest.aio.assert_status_code(response, 200) + assert response['Item']['testKey'] == {'S': test_value}