From a1903aeaf9051cfae77c41748de99310b5041ad3 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Sat, 8 Nov 2014 13:09:12 -0500 Subject: [PATCH 1/3] Move 'bucket.BucketIterator' -> 'connection._BucketIterator'. Rename to indicate the iterator is an implementation detail of 'Connection'. --- gcloud/storage/bucket.py | 23 ------------- gcloud/storage/connection.py | 27 ++++++++++++++-- gcloud/storage/test_bucket.py | 36 --------------------- gcloud/storage/test_connection.py | 54 +++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 61 deletions(-) diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index 8f9135cdea88..ab1c27cb87c1 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -643,26 +643,3 @@ def make_public(self, recursive=False, future=False): for key in self: key.get_acl().all().grant_read() key.save_acl() - - -class BucketIterator(Iterator): - """An iterator listing all buckets. - - You shouldn't have to use this directly, but instead should use the helper - methods on :class:`gcloud.storage.connection.Connection` objects. - - :type connection: :class:`gcloud.storage.connection.Connection` - :param connection: The connection to use for querying the list of buckets. - """ - - def __init__(self, connection): - super(BucketIterator, self).__init__(connection=connection, path='/b') - - def get_items_from_response(self, response): - """Factory method which yields :class:`.Bucket` items from a response. - - :type response: dict - :param response: The JSON API response for a page of buckets. - """ - for item in response.get('items', []): - yield Bucket.from_dict(item, connection=self.connection) diff --git a/gcloud/storage/connection.py b/gcloud/storage/connection.py index 00eb63772cac..d32f438c55f6 100644 --- a/gcloud/storage/connection.py +++ b/gcloud/storage/connection.py @@ -15,7 +15,7 @@ from gcloud import connection from gcloud.storage import exceptions from gcloud.storage.bucket import Bucket -from gcloud.storage.bucket import BucketIterator +from gcloud.storage.iterator import Iterator def _utcnow(): # pragma: NO COVER testing replaces @@ -84,7 +84,7 @@ def __init__(self, project, *args, **kwargs): self.project = project def __iter__(self): - return iter(BucketIterator(connection=self)) + return iter(_BucketIterator(connection=self)) def __contains__(self, bucket_name): return self.lookup(bucket_name) is not None @@ -488,6 +488,29 @@ def generate_signed_url(self, resource, expiration, querystring=urllib.urlencode(query_params)) +class _BucketIterator(Iterator): + """An iterator listing all buckets. + + You shouldn't have to use this directly, but instead should use the helper + methods on :class:`gcloud.storage.connection.Connection` objects. + + :type connection: :class:`gcloud.storage.connection.Connection` + :param connection: The connection to use for querying the list of buckets. + """ + + def __init__(self, connection): + super(_BucketIterator, self).__init__(connection=connection, path='/b') + + def get_items_from_response(self, response): + """Factory method which yields :class:`.Bucket` items from a response. + + :type response: dict + :param response: The JSON API response for a page of buckets. + """ + for item in response.get('items', []): + yield Bucket.from_dict(item, connection=self.connection) + + def _get_expiration_seconds(expiration): """Convert 'expiration' to a number of seconds in the future. diff --git a/gcloud/storage/test_bucket.py b/gcloud/storage/test_bucket.py index 7e4dbe6f65d9..948c0dba265c 100644 --- a/gcloud/storage/test_bucket.py +++ b/gcloud/storage/test_bucket.py @@ -884,42 +884,6 @@ def get_items_from_response(self, response): self.assertEqual(kw[1]['query_params'], {}) -class TestBucketIterator(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.storage.bucket import BucketIterator - return BucketIterator - - def _makeOne(self, *args, **kw): - return self._getTargetClass()(*args, **kw) - - def test_ctor(self): - connection = _Connection() - iterator = self._makeOne(connection) - self.assertTrue(iterator.connection is connection) - self.assertEqual(iterator.path, '/b') - self.assertEqual(iterator.page_number, 0) - self.assertEqual(iterator.next_page_token, None) - - def test_get_items_from_response_empty(self): - connection = _Connection() - iterator = self._makeOne(connection) - self.assertEqual(list(iterator.get_items_from_response({})), []) - - def test_get_items_from_response_non_empty(self): - from gcloud.storage.bucket import Bucket - KEY = 'key' - response = {'items': [{'name': KEY}]} - connection = _Connection() - iterator = self._makeOne(connection) - buckets = list(iterator.get_items_from_response(response)) - self.assertEqual(len(buckets), 1) - bucket = buckets[0] - self.assertTrue(isinstance(bucket, Bucket)) - self.assertTrue(bucket.connection is connection) - self.assertEqual(bucket.name, KEY) - - class _Connection(object): _delete_ok = False diff --git a/gcloud/storage/test_connection.py b/gcloud/storage/test_connection.py index 7987cb40266c..37a21833aba8 100644 --- a/gcloud/storage/test_connection.py +++ b/gcloud/storage/test_connection.py @@ -610,6 +610,42 @@ def test_generate_signed_url_w_expiration_int(self): self.assertEqual(frag, '') +class Test__BucketIterator(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.storage.connection import _BucketIterator + return _BucketIterator + + def _makeOne(self, *args, **kw): + return self._getTargetClass()(*args, **kw) + + def test_ctor(self): + connection = _Connection() + iterator = self._makeOne(connection) + self.assertTrue(iterator.connection is connection) + self.assertEqual(iterator.path, '/b') + self.assertEqual(iterator.page_number, 0) + self.assertEqual(iterator.next_page_token, None) + + def test_get_items_from_response_empty(self): + connection = _Connection() + iterator = self._makeOne(connection) + self.assertEqual(list(iterator.get_items_from_response({})), []) + + def test_get_items_from_response_non_empty(self): + from gcloud.storage.bucket import Bucket + KEY = 'key' + response = {'items': [{'name': KEY}]} + connection = _Connection() + iterator = self._makeOne(connection) + buckets = list(iterator.get_items_from_response(response)) + self.assertEqual(len(buckets), 1) + bucket = buckets[0] + self.assertTrue(isinstance(bucket, Bucket)) + self.assertTrue(bucket.connection is connection) + self.assertEqual(bucket.name, KEY) + + class Test__get_expiration_seconds(unittest2.TestCase): def _callFUT(self, expiration): @@ -705,6 +741,24 @@ def request(self, **kw): return self._response, self._content +class _Connection(object): + + def __init__(self, *responses): + self._responses = responses + self._requested = [] + + def api_request(self, **kw): + from gcloud.storage.exceptions import NotFound + self._requested.append(kw) + + try: + response, self._responses = self._responses[0], self._responses[1:] + except: + raise NotFound('miss') + else: + return response + + class _Credentials(object): service_account_name = 'testing@example.com' From 47b23fbe49ebdbb6b710c62d5b7322f8665a7d0f Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Sat, 8 Nov 2014 13:33:25 -0500 Subject: [PATCH 2/3] Unneeded shim. --- gcloud/storage/test_connection.py | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/gcloud/storage/test_connection.py b/gcloud/storage/test_connection.py index 37a21833aba8..d5a0eb2d5694 100644 --- a/gcloud/storage/test_connection.py +++ b/gcloud/storage/test_connection.py @@ -620,7 +620,7 @@ def _makeOne(self, *args, **kw): return self._getTargetClass()(*args, **kw) def test_ctor(self): - connection = _Connection() + connection = object() iterator = self._makeOne(connection) self.assertTrue(iterator.connection is connection) self.assertEqual(iterator.path, '/b') @@ -628,7 +628,7 @@ def test_ctor(self): self.assertEqual(iterator.next_page_token, None) def test_get_items_from_response_empty(self): - connection = _Connection() + connection = object() iterator = self._makeOne(connection) self.assertEqual(list(iterator.get_items_from_response({})), []) @@ -636,7 +636,7 @@ def test_get_items_from_response_non_empty(self): from gcloud.storage.bucket import Bucket KEY = 'key' response = {'items': [{'name': KEY}]} - connection = _Connection() + connection = object() iterator = self._makeOne(connection) buckets = list(iterator.get_items_from_response(response)) self.assertEqual(len(buckets), 1) @@ -741,24 +741,6 @@ def request(self, **kw): return self._response, self._content -class _Connection(object): - - def __init__(self, *responses): - self._responses = responses - self._requested = [] - - def api_request(self, **kw): - from gcloud.storage.exceptions import NotFound - self._requested.append(kw) - - try: - response, self._responses = self._responses[0], self._responses[1:] - except: - raise NotFound('miss') - else: - return response - - class _Credentials(object): service_account_name = 'testing@example.com' From d0cff2c13993b79b9bba27bff36754163947ca49 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Sat, 8 Nov 2014 13:34:19 -0500 Subject: [PATCH 3/3] Move 'key._KeyIterator' -> 'bucket._KeyIterator'. The iterator is an implementation detail of 'Bucket'. --- gcloud/storage/bucket.py | 32 +++++++++++++++++-- gcloud/storage/connection.py | 4 +-- gcloud/storage/key.py | 26 ---------------- gcloud/storage/test_bucket.py | 58 +++++++++++++++++++++++++++++++---- gcloud/storage/test_key.py | 40 ------------------------ 5 files changed, 83 insertions(+), 77 deletions(-) diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index ab1c27cb87c1..f60df5c6a26c 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -9,7 +9,31 @@ from gcloud.storage.acl import DefaultObjectACL from gcloud.storage.iterator import Iterator from gcloud.storage.key import Key -from gcloud.storage.key import _KeyIterator + + +class _KeyIterator(Iterator): + """An iterator listing keys in a bucket + + You shouldn't have to use this directly, but instead should use the + helper methods on :class:`gcloud.storage.key.Bucket` objects. + + :type bucket: :class:`gcloud.storage.bucket.Bucket` + :param bucket: The bucket from which to list keys. + """ + def __init__(self, bucket, connection=None, extra_params=None): + self.bucket = bucket + super(_KeyIterator, self).__init__( + connection=bucket.connection, path=bucket.path + '/o', + extra_params=extra_params) + + def get_items_from_response(self, response): + """Yield :class:`.storage.key.Key` items from response. + + :type response: dict + :param response: The JSON API response for a page of keys. + """ + for item in response.get('items', []): + yield Key.from_dict(item, bucket=self.bucket) class Bucket(_PropertyMixin): @@ -21,6 +45,7 @@ class Bucket(_PropertyMixin): :type name: string :param name: The name of the bucket. """ + _iterator_class = _KeyIterator CUSTOM_PROPERTY_ACCESSORS = { 'acl': 'get_acl()', @@ -66,7 +91,7 @@ def __repr__(self): return '' % self.name def __iter__(self): - return iter(_KeyIterator(bucket=self)) + return iter(self._iterator_class(bucket=self)) def __contains__(self, key): return self.get_key(key) is not None @@ -640,6 +665,7 @@ def make_public(self, recursive=False, future=False): doa.save() if recursive: - for key in self: + iterator = self._iterator_class(self) + for key in iterator: key.get_acl().all().grant_read() key.save_acl() diff --git a/gcloud/storage/connection.py b/gcloud/storage/connection.py index d32f438c55f6..08f492312568 100644 --- a/gcloud/storage/connection.py +++ b/gcloud/storage/connection.py @@ -12,7 +12,7 @@ from OpenSSL import crypto import pytz -from gcloud import connection +from gcloud.connection import Connection as _Base from gcloud.storage import exceptions from gcloud.storage.bucket import Bucket from gcloud.storage.iterator import Iterator @@ -26,7 +26,7 @@ def _utcnow(): # pragma: NO COVER testing replaces return datetime.datetime.utcnow() -class Connection(connection.Connection): +class Connection(_Base): """A connection to Google Cloud Storage via the JSON REST API. This class should understand only the basic types (and protobufs) diff --git a/gcloud/storage/key.py b/gcloud/storage/key.py index 8104f5313953..abfa325ba27e 100644 --- a/gcloud/storage/key.py +++ b/gcloud/storage/key.py @@ -9,7 +9,6 @@ from gcloud.storage._helpers import _scalar_property from gcloud.storage.acl import ObjectACL from gcloud.storage.exceptions import StorageError -from gcloud.storage.iterator import Iterator class Key(_PropertyMixin): @@ -595,31 +594,6 @@ def updated(self): return self.properties['updated'] -class _KeyIterator(Iterator): - """An iterator listing keys. - - You shouldn't have to use this directly, but instead should use the - helper methods on :class:`gcloud.storage.key.Key` objects. - - :type bucket: :class:`gcloud.storage.bucket.Bucket` - :param bucket: The bucket from which to list keys. - """ - def __init__(self, bucket, extra_params=None): - self.bucket = bucket - super(_KeyIterator, self).__init__( - connection=bucket.connection, path=bucket.path + '/o', - extra_params=extra_params) - - def get_items_from_response(self, response): - """Yield :class:`.storage.key.Key` items from response. - - :type response: dict - :param response: The JSON API response for a page of keys. - """ - for item in response.get('items', []): - yield Key.from_dict(item, bucket=self.bucket) - - class _KeyDataIterator(object): """An iterator listing data stored in a key. diff --git a/gcloud/storage/test_bucket.py b/gcloud/storage/test_bucket.py index 948c0dba265c..a8464063eb1f 100644 --- a/gcloud/storage/test_bucket.py +++ b/gcloud/storage/test_bucket.py @@ -832,9 +832,7 @@ def test_make_public_w_future(self): def test_make_public_recursive(self): from gcloud.storage.acl import _ACLEntity - from gcloud._testing import _Monkey - from gcloud.storage import key - from gcloud.storage import bucket as MUT + from gcloud.storage.bucket import _KeyIterator _saved = [] class _Key(object): @@ -856,7 +854,7 @@ def grant_read(self): def save_acl(self): _saved.append((self._bucket, self._name, self._granted)) - class _KeyIterator(key._KeyIterator): + class _Iterator(_KeyIterator): def get_items_from_response(self, response): for item in response.get('items', []): yield _Key(self.bucket, item['name']) @@ -868,8 +866,8 @@ def get_items_from_response(self, response): connection = _Connection(after, {'items': [{'name': KEY}]}) bucket = self._makeOne(connection, NAME) bucket.acl.loaded = True - with _Monkey(MUT, _KeyIterator=_KeyIterator): - bucket.make_public(recursive=True) + bucket._iterator_class = _Iterator + bucket.make_public(recursive=True) self.assertEqual(list(bucket.acl), permissive) self.assertEqual(list(bucket.default_object_acl), []) self.assertEqual(_saved, [(bucket, KEY, True)]) @@ -884,6 +882,46 @@ def get_items_from_response(self, response): self.assertEqual(kw[1]['query_params'], {}) +class Test__KeyIterator(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.storage.bucket import _KeyIterator + return _KeyIterator + + def _makeOne(self, *args, **kw): + return self._getTargetClass()(*args, **kw) + + def test_ctor(self): + connection = _Connection() + bucket = _Bucket(connection) + iterator = self._makeOne(bucket) + self.assertTrue(iterator.bucket is bucket) + self.assertTrue(iterator.connection is connection) + self.assertEqual(iterator.path, '%s/o' % bucket.path) + self.assertEqual(iterator.page_number, 0) + self.assertEqual(iterator.next_page_token, None) + + def test_get_items_from_response_empty(self): + connection = _Connection() + bucket = _Bucket(connection) + iterator = self._makeOne(bucket) + self.assertEqual(list(iterator.get_items_from_response({})), []) + + def test_get_items_from_response_non_empty(self): + from gcloud.storage.key import Key + KEY = 'key' + response = {'items': [{'name': KEY}]} + connection = _Connection() + bucket = _Bucket(connection) + iterator = self._makeOne(bucket) + keys = list(iterator.get_items_from_response(response)) + self.assertEqual(len(keys), 1) + key = keys[0] + self.assertTrue(isinstance(key, Key)) + self.assertTrue(key.connection is connection) + self.assertEqual(key.name, KEY) + + class _Connection(object): _delete_ok = False @@ -911,6 +949,14 @@ def delete_bucket(self, bucket, force=False): return True +class _Bucket(object): + path = '/b/name' + name = 'name' + + def __init__(self, connection): + self.connection = connection + + class MockFile(io.StringIO): name = None diff --git a/gcloud/storage/test_key.py b/gcloud/storage/test_key.py index be6e4adbabc7..7f330a3903f7 100644 --- a/gcloud/storage/test_key.py +++ b/gcloud/storage/test_key.py @@ -662,46 +662,6 @@ def test_updated(self): self.assertEqual(key.updated, UPDATED) -class Test__KeyIterator(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.storage.key import _KeyIterator - return _KeyIterator - - def _makeOne(self, *args, **kw): - return self._getTargetClass()(*args, **kw) - - def test_ctor(self): - connection = _Connection() - bucket = _Bucket(connection) - iterator = self._makeOne(bucket) - self.assertTrue(iterator.bucket is bucket) - self.assertTrue(iterator.connection is connection) - self.assertEqual(iterator.path, '%s/o' % bucket.path) - self.assertEqual(iterator.page_number, 0) - self.assertEqual(iterator.next_page_token, None) - - def test_get_items_from_response_empty(self): - connection = _Connection() - bucket = _Bucket(connection) - iterator = self._makeOne(bucket) - self.assertEqual(list(iterator.get_items_from_response({})), []) - - def test_get_items_from_response_non_empty(self): - from gcloud.storage.key import Key - KEY = 'key' - response = {'items': [{'name': KEY}]} - connection = _Connection() - bucket = _Bucket(connection) - iterator = self._makeOne(bucket) - keys = list(iterator.get_items_from_response(response)) - self.assertEqual(len(keys), 1) - key = keys[0] - self.assertTrue(isinstance(key, Key)) - self.assertTrue(key.connection is connection) - self.assertEqual(key.name, KEY) - - class Test__KeyDataIterator(unittest2.TestCase): def _getTargetClass(self):