From c43362b8b4440a82640297dfc8adcd76852b7661 Mon Sep 17 00:00:00 2001 From: "Bala.FA" Date: Thu, 6 Aug 2020 21:01:26 +0530 Subject: [PATCH] merge list_objects_v2() into list_objects() method --- README.md | 3 - docs/API.md | 68 +-------- examples/get_partial_object.py | 34 ----- examples/list_objects.py | 4 +- examples/remove_objects.py | 2 +- minio/__init__.py | 2 +- minio/api.py | 73 ++-------- tests/functional/tests.py | 42 +++--- tests/unit/list_objects_test.py | 182 +++++------------------ tests/unit/list_objects_v1_test.py | 222 +++++++++++++++++++++++++++++ tests/unit/list_objects_v2_test.py | 102 ------------- 11 files changed, 298 insertions(+), 436 deletions(-) delete mode 100644 examples/get_partial_object.py create mode 100644 tests/unit/list_objects_v1_test.py delete mode 100644 tests/unit/list_objects_v2_test.py diff --git a/README.md b/README.md index 5f7bf4800..663f41351 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,6 @@ The full API Reference is available here. * [`bucket_exists`](https://docs.min.io/docs/python-client-api-reference#bucket_exists) * [`remove_bucket`](https://docs.min.io/docs/python-client-api-reference#remove_bucket) * [`list_objects`](https://docs.min.io/docs/python-client-api-reference#list_objects) -* [`list_objects_v2`](https://docs.min.io/docs/python-client-api-reference#list_objects_v2) * [`list_incomplete_uploads`](https://docs.min.io/docs/python-client-api-reference#list_incomplete_uploads) ### API Reference : Bucket policy Operations @@ -144,7 +143,6 @@ The full API Reference is available here. * [`put_object`](https://docs.min.io/docs/python-client-api-reference#put_object) * [`stat_object`](https://docs.min.io/docs/python-client-api-reference#stat_object) * [`copy_object`](https://docs.min.io/docs/python-client-api-reference#copy_object) -* [`get_partial_object`](https://docs.min.io/docs/python-client-api-reference#get_partial_object) * [`remove_object`](https://docs.min.io/docs/python-client-api-reference#remove_object) * [`remove_objects`](https://docs.min.io/docs/python-client-api-reference#remove_objects) * [`remove_incomplete_upload`](https://docs.min.io/docs/python-client-api-reference#remove_incomplete_upload) @@ -195,7 +193,6 @@ The full API Reference is available here. * [put_object.py](https://github.com/minio/minio-py/blob/master/examples/put_object.py) * [stat_object.py](https://github.com/minio/minio-py/blob/master/examples/stat_object.py) * [copy_object.py](https://github.com/minio/minio-py/blob/master/examples/copy_object.py) -* [get_partial_object.py](https://github.com/minio/minio-py/blob/master/examples/get_partial_object.py) * [remove_object.py](https://github.com/minio/minio-py/blob/master/examples/remove_object.py) * [remove_objects.py](https://github.com/minio/minio-py/blob/master/examples/remove_objects.py) * [remove_incomplete_upload.py](https://github.com/minio/minio-py/blob/master/examples/remove_incomplete_upload.py) diff --git a/docs/API.md b/docs/API.md index 27ebe273c..4222f0067 100644 --- a/docs/API.md +++ b/docs/API.md @@ -37,10 +37,10 @@ s3Client = Minio( | [`bucket_exists`](#bucket_exists) | [`copy_object`](#copy_object) | [`presigned_post_policy`](#presigned_post_policy) | [`delete_bucket_policy`](#delete_bucket_policy) | | [`remove_bucket`](#remove_bucket) | [`stat_object`](#stat_object) | | [`get_bucket_notification`](#get_bucket_notification) | | [`list_objects`](#list_objects) | [`remove_object`](#remove_object) | | [`set_bucket_notification`](#set_bucket_notification) | -| [`list_objects_v2`](#list_objects_v2) | [`remove_objects`](#remove_objects) | | [`remove_all_bucket_notification`](#remove_all_bucket_notification) | -| [`list_incomplete_uploads`](#list_incomplete_uploads) | [`remove_incomplete_upload`](#remove_incomplete_upload) | | [`listen_bucket_notification`](#listen_bucket_notification) | -| [`enable_bucket_versioning`](#enable_bucket_versioning) | [`fput_object`](#fput_object) | | [`get_bucket_encryption`](#get_bucket_encryption) | -| [`disable_bucket_versioning`](#disable_bucket_versioning) | [`fget_object`](#fget_object) | | [`put_bucket_encryption`](#put_bucket_encryption) | +| [`list_incomplete_uploads`](#list_incomplete_uploads) | [`remove_objects`](#remove_objects) | | [`remove_all_bucket_notification`](#remove_all_bucket_notification) | +| [`enable_bucket_versioning`](#enable_bucket_versioning) | [`remove_incomplete_upload`](#remove_incomplete_upload) | | [`listen_bucket_notification`](#listen_bucket_notification) | +| [`disable_bucket_versioning`](#disable_bucket_versioning) | [`fput_object`](#fput_object) | | [`get_bucket_encryption`](#get_bucket_encryption) | +| | [`fget_object`](#fget_object) | | [`put_bucket_encryption`](#put_bucket_encryption) | | | [`select_object_content`](#select_object_content) | | [`delete_bucket_encryption`](#delete_bucket_encryption) | ## 1. Constructor @@ -256,64 +256,6 @@ for object in objects: print(object) ``` - - -### list_objects_v2(bucket_name, prefix=None, recursive=False, start_after=None, include_user_meta=False, include_version=False) - -Lists object information of a bucket using S3 API version 2, optionally for prefix recursively. - -__Parameters__ - -| Param | Type | Description | -|:--------------------|:-------|:---------------------------------------------------------| -| `bucket_name` | _str_ | Name of the bucket. | -| `prefix` | _str_ | Object name starts with prefix. | -| `recursive` | _bool_ | List recursively than directory structure emulation. | -| `start_after` | _str_ | List objects after this key name. | -| `include_user_meta` | _bool_ | MinIO specific flag to control to include user metadata. | -| `include_version` | _bool_ | Flag to control whether include object versions. | - -__Return Value__ - -| Return | -|:----------------------------------------------------------| -| An iterator contains object information as _minio.Object_ | - -__Example__ - -```py -# List objects information. -objects = minio.list_objects_v2('foo') -for object in objects: - print(object) - -# List objects information those names starts with 'hello/'. -objects = minio.list_objects_v2('foo', prefix='hello/') -for object in objects: - print(object) - -# List objects information recursively. -objects = minio.list_objects_v2('foo', recursive=True) -for object in objects: - print(object) - -# List objects information recursively those names starts with -# 'hello/'. -objects = minio.list_objects_v2( - 'foo', prefix='hello/', recursive=True, -) -for object in objects: - print(object) - -# List objects information recursively after object name -# 'hello/world/1'. -objects = minio.list_objects_v2( - 'foo', recursive=True, start_after='hello/world/1', -) -for object in objects: - print(object) -``` - ### list_incomplete_uploads(bucket_name, prefix='', recursive=False) @@ -752,7 +694,7 @@ finally: // Get object data for offset/length. try: - response = minio.get_partial_object('foo', 'bar', 2, 4) + response = minio.get_object('foo', 'bar', 2, 4) // Read data from response. finally: response.close() diff --git a/examples/get_partial_object.py b/examples/get_partial_object.py deleted file mode 100644 index 6aaec1118..000000000 --- a/examples/get_partial_object.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- -# MinIO Python Library for Amazon S3 Compatible Cloud Storage, -# (C) 2015 MinIO, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname -# and my-testfile are dummy values, please replace them with original values. - -from minio import Minio -from minio.error import ResponseError - -client = Minio('s3.amazonaws.com', - access_key='YOUR-ACCESSKEYID', - secret_key='YOUR-SECRETACCESSKEY') - -# Offset the download by 2 bytes and retrieve a total of 4 bytes. -try: - data = client.get_partial_object('my-bucketname', 'my-objectname', 2, 4) - with open('my-testfile', 'wb') as file_data: - for d in data: - file_data.write(d) -except ResponseError as err: - print(err) diff --git a/examples/list_objects.py b/examples/list_objects.py index 53c6c6746..83ca7af03 100644 --- a/examples/list_objects.py +++ b/examples/list_objects.py @@ -32,8 +32,8 @@ # List all object paths in bucket that begin with my-prefixname using # V2 listing API. -objects = client.list_objects_v2('my-bucketname', prefix='my-prefixname', - recursive=True) +objects = client.list_objects('my-bucketname', prefix='my-prefixname', + recursive=True) for obj in objects: print(obj.bucket_name, obj.object_name.encode('utf-8'), obj.last_modified, obj.etag, obj.size, obj.content_type) diff --git a/examples/remove_objects.py b/examples/remove_objects.py index ab2618ffe..aa47a68a2 100644 --- a/examples/remove_objects.py +++ b/examples/remove_objects.py @@ -28,7 +28,7 @@ try: names = map( lambda x: x.object_name, - client.list_objects_v2('my-bucketname', 'my-prefix', recursive=True) + client.list_objects('my-bucketname', 'my-prefix', recursive=True) ) for err in client.remove_objects('my-bucketname', names): print("Deletion Error: {}".format(err)) diff --git a/minio/__init__.py b/minio/__init__.py index b936008a1..76002bf4c 100644 --- a/minio/__init__.py +++ b/minio/__init__.py @@ -29,7 +29,7 @@ __title__ = 'minio-py' __author__ = 'MinIO, Inc.' -__version__ = '6.0.1' +__version__ = '7.0.0' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2015, 2016, 2017, 2018, 2019, 2020 MinIO, Inc.' diff --git a/minio/api.py b/minio/api.py index bd75970a0..5017ea047 100644 --- a/minio/api.py +++ b/minio/api.py @@ -871,7 +871,7 @@ def get_object(self, bucket_name, object_name, offset=0, length=0, // Get object data for offset/length. try: - response = minio.get_partial_object('foo', 'bar', 2, 4) + response = minio.get_object('foo', 'bar', 2, 4) // Read data from response. finally: response.close() @@ -1039,53 +1039,8 @@ def put_object(self, bucket_name, object_name, data, length, progress=progress) def list_objects(self, bucket_name, prefix=None, recursive=False, - include_version=False): - """ - Lists object information of a bucket using S3 API version 1, optionally - for prefix recursively. - - :param bucket_name: Name of the bucket. - :param prefix: Object name starts with prefix. - :param recursive: List recursively than directory structure emulation. - :param include_version: Flag to control whether include object - versions. - :return: An iterator contains object information. - - Example:: - # List objects information. - objects = minio.list_objects('foo') - for object in objects: - print(object) - - # List objects information those names starts with 'hello/'. - objects = minio.list_objects('foo', prefix='hello/') - for object in objects: - print(object) - - # List objects information recursively. - objects = minio.list_objects('foo', recursive=True) - for object in objects: - print(object) - - # List objects information recursively those names starts with - # 'hello/'. - objects = minio.list_objects( - 'foo', prefix='hello/', recursive=True, - ) - for object in objects: - print(object) - """ - return self._list_objects( - bucket_name, - delimiter=None if recursive else "/", - prefix=prefix, - use_version1=True, - include_version=include_version, - ) - - def list_objects_v2(self, bucket_name, prefix=None, recursive=False, - start_after=None, include_user_meta=False, - include_version=False): + start_after=None, include_user_meta=False, + include_version=False, use_api_v1=False): """ Lists object information of a bucket using S3 API version 2, optionally for prefix recursively. @@ -1098,27 +1053,28 @@ def list_objects_v2(self, bucket_name, prefix=None, recursive=False, user metadata. :param include_version: Flag to control whether include object versions. + :param use_api_v1: Flag to control to use ListObjectV1 S3 API or not. :return: An iterator contains object information. Example:: # List objects information. - objects = minio.list_objects_v2('foo') + objects = minio.list_objects('foo') for object in objects: print(object) # List objects information those names starts with 'hello/'. - objects = minio.list_objects_v2('foo', prefix='hello/') + objects = minio.list_objects('foo', prefix='hello/') for object in objects: print(object) # List objects information recursively. - objects = minio.list_objects_v2('foo', recursive=True) + objects = minio.list_objects('foo', recursive=True) for object in objects: print(object) # List objects information recursively those names starts with # 'hello/'. - objects = minio.list_objects_v2( + objects = minio.list_objects( 'foo', prefix='hello/', recursive=True, ) for object in objects: @@ -1126,7 +1082,7 @@ def list_objects_v2(self, bucket_name, prefix=None, recursive=False, # List objects information recursively after object name # 'hello/world/1'. - objects = minio.list_objects_v2( + objects = minio.list_objects( 'foo', recursive=True, start_after='hello/world/1', ) for object in objects: @@ -1138,6 +1094,7 @@ def list_objects_v2(self, bucket_name, prefix=None, recursive=False, include_user_meta=include_user_meta, prefix=prefix, start_after=start_after, + use_api_v1=use_api_v1, include_version=include_version, ) @@ -2151,7 +2108,7 @@ def _list_objects( # pylint: disable=too-many-arguments,too-many-branches prefix=None, # all start_after=None, # all: v1:marker, versioned:key_marker version_id_marker=None, # versioned - use_version1=False, + use_api_v1=False, include_version=False, ): """ @@ -2172,10 +2129,10 @@ def _list_objects( # pylint: disable=too-many-arguments,too-many-branches query = {} if include_version: query["versions"] = "" - elif not use_version1: + elif not use_api_v1: query["list-type"] = "2" - if not include_version and not use_version1: + if not include_version and not use_api_v1: if continuation_token: query["continuation-token"] = continuation_token if fetch_owner: @@ -2190,7 +2147,7 @@ def _list_objects( # pylint: disable=too-many-arguments,too-many-branches if start_after: if include_version: query["key-marker"] = start_after - elif use_version1: + elif use_api_v1: query["marker"] = start_after else: query["start-after"] = start_after @@ -2207,7 +2164,7 @@ def _list_objects( # pylint: disable=too-many-arguments,too-many-branches objects, is_truncated, start_after, version_id_marker = ( parse_list_object_versions(response.data, bucket_name) ) - elif use_version1: + elif use_api_v1: objects, is_truncated, start_after = parse_list_objects( response.data, bucket_name, diff --git a/tests/functional/tests.py b/tests/functional/tests.py index bad060e2d..33af01458 100644 --- a/tests/functional/tests.py +++ b/tests/functional/tests.py @@ -1064,7 +1064,7 @@ def test_get_partial_object(log_entry, sse=None): _CLIENT.remove_bucket(bucket_name) -def _test_list_objects(log_entry, version2=False, version_check=False): +def _test_list_objects(log_entry, use_api_v1=False, version_check=False): """Test list_objects().""" # Get a unique bucket_name and object_name @@ -1092,14 +1092,10 @@ def _test_list_objects(log_entry, version2=False, version_check=False): bucket_name, object_name + "-2", LimitedRandomReader(size), size, ) # List all object paths in bucket. - if version2: - objects = _CLIENT.list_objects_v2( - bucket_name, '', is_recursive, include_version=version_check, - ) - else: - objects = _CLIENT.list_objects( - bucket_name, '', is_recursive, include_version=version_check, - ) + objects = _CLIENT.list_objects( + bucket_name, '', is_recursive, include_version=version_check, + use_api_v1=use_api_v1, + ) for obj in objects: _ = (obj.bucket_name, obj.object_name, obj.last_modified, obj.etag, obj.size, obj.content_type) @@ -1119,14 +1115,14 @@ def _test_list_objects(log_entry, version2=False, version_check=False): _CLIENT.remove_bucket(bucket_name) -def test_list_objects(log_entry): +def test_list_objects_v1(log_entry): """Test list_objects().""" - _test_list_objects(log_entry) + _test_list_objects(log_entry, use_api_v1=True) -def test_list_object_versions(log_entry): +def test_list_object_v1_versions(log_entry): """Test list_objects().""" - _test_list_objects(log_entry, version_check=True) + _test_list_objects(log_entry, use_api_v1=True, version_check=True) def _test_list_objects_api(bucket_name, expected_no, *argv): @@ -1266,14 +1262,14 @@ def test_list_objects_with_1001_files( # pylint: disable=invalid-name _CLIENT.remove_bucket(bucket_name) -def test_list_objects_v2(log_entry): - """Test list_objects_v2().""" - _test_list_objects(log_entry, version2=True) +def test_list_objects(log_entry): + """Test list_objects().""" + _test_list_objects(log_entry) -def test_list_object_versions_v2(log_entry): - """Test list_objects_v2() of versioned object.""" - _test_list_objects(log_entry, version2=True, version_check=True) +def test_list_object_versions(log_entry): + """Test list_objects() of versioned object.""" + _test_list_objects(log_entry, version_check=True) def _create_upload_ids(bucket_name, object_name, count): @@ -1982,13 +1978,13 @@ def main(): test_fget_object_version: {"sse": ssec} if ssec else None, test_get_object_with_default_length: None, test_get_partial_object: {"sse": ssec} if ssec else None, - test_list_objects: None, - test_list_object_versions: None, + test_list_objects_v1: None, + test_list_object_v1_versions: None, test_list_objects_with_prefix: None, test_list_objects_with_1001_files: None, test_remove_incomplete_upload: None, - test_list_objects_v2: None, - test_list_object_versions_v2: None, + test_list_objects: None, + test_list_object_versions: None, test_presigned_get_object_default_expiry: None, test_presigned_get_object_expiry: None, test_presigned_get_object_response_headers: None, diff --git a/tests/unit/list_objects_test.py b/tests/unit/list_objects_test.py index a807bf729..eb4f992fc 100644 --- a/tests/unit/list_objects_test.py +++ b/tests/unit/list_objects_test.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # MinIO Python Library for Amazon S3 Compatible Cloud Storage, -# (C) 2015, 2016 MinIO, Inc. +# (C) 2015-2020 MinIO, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -28,66 +28,57 @@ class ListObjectsTest(TestCase): @mock.patch('urllib3.PoolManager') def test_empty_list_objects_works(self, mock_connection): - mock_data = ''' + mock_data = ''' bucket - - - false + + 0 1000 - + + false ''' mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request( MockResponse( "GET", - "https://localhost:9000/bucket/" - "?delimiter=&max-keys=1000&prefix=", + "https://localhost:9000/bucket/?delimiter=&list-type=2" + "&max-keys=1000&prefix=", {"User-Agent": _DEFAULT_USER_AGENT}, 200, content=mock_data, ), ) client = Minio('localhost:9000') - bucket_iter = client.list_objects('bucket', recursive=True) - buckets = [] - for bucket in bucket_iter: - buckets.append(bucket) - eq_(0, len(buckets)) + object_iter = client.list_objects('bucket', recursive=True) + objects = [] + for obj in object_iter: + objects.append(obj) + eq_(0, len(objects)) @timed(1) @mock.patch('urllib3.PoolManager') def test_list_objects_works(self, mock_connection): - mock_data = ''' + mock_data = ''' bucket - - + + 2 1000 - false - key1 - 2015-05-05T02:21:15.716Z - 5eb63bbbe01eeed093cb22bb8f5acdc3 - 11 - STANDARD - - minio - minio - + 6/f/9/6f9898076bb08572403f95dbb86c5b9c85e1e1b3 + 2016-11-27T07:55:53.000Z + "5d5512301b6b6e247b8aec334b2cf7ea" + 493 + REDUCED_REDUNDANCY - key2 - 2015-05-05T20:36:17.498Z - 2a60eaffa7a82804bdc682ce1df6c2d4 - 1661 - STANDARD - - minio - minio - + b/d/7/bd7f6410cced55228902d881c2954ebc826d7464 + 2016-11-27T07:10:27.000Z + "f00483d523ffc8b7f2883ae896769d85" + 493 + REDUCED_REDUNDANCY ''' mock_server = MockConnection() @@ -95,124 +86,17 @@ def test_list_objects_works(self, mock_connection): mock_server.mock_add_request( MockResponse( "GET", - "https://localhost:9000/bucket/" - "?delimiter=%2F&max-keys=1000&prefix=", + "https://localhost:9000/bucket/?delimiter=%2F&list-type=2" + "&max-keys=1000&prefix=", {"User-Agent": _DEFAULT_USER_AGENT}, 200, content=mock_data, ), ) client = Minio('localhost:9000') - bucket_iter = client.list_objects('bucket') - buckets = [] - for bucket in bucket_iter: - # cause an xml exception and fail if we try retrieving again - mock_server.mock_add_request( - MockResponse( - "GET", - "https://localhost:9000/bucket/" - "?delimiter=%2F&max-keys=1000&prefix=", - {"User-Agent": _DEFAULT_USER_AGENT}, - 200, - content="", - ), - ) - buckets.append(bucket) - - eq_(2, len(buckets)) - - @timed(1) - @mock.patch('urllib3.PoolManager') - def test_list_objects_works_well(self, mock_connection): - mock_data1 = ''' - - bucket - - - marker - 1000 - - true - - key1 - 2015-05-05T02:21:15.716Z - 5eb63bbbe01eeed093cb22bb8f5acdc3 - 11 - STANDARD - - minio - minio - - - - key2 - 2015-05-05T20:36:17.498Z - 2a60eaffa7a82804bdc682ce1df6c2d4 - 1661 - STANDARD - - minio - minio - - -''' - mock_data2 = ''' - - bucket - - - 1000 - - false - - key3 - 2015-05-05T02:21:15.716Z - 5eb63bbbe01eeed093cb22bb8f5acdc3 - 11 - STANDARD - - minio - minio - - - - key4 - 2015-05-05T20:36:17.498Z - 2a60eaffa7a82804bdc682ce1df6c2d4 - 1661 - STANDARD - - minio - minio - - -''' - mock_server = MockConnection() - mock_connection.return_value = mock_server - mock_server.mock_add_request( - MockResponse( - "GET", - "https://localhost:9000/bucket/" - "?delimiter=&max-keys=1000&prefix=", - {"User-Agent": _DEFAULT_USER_AGENT}, - 200, - content=mock_data1, - ), - ) - client = Minio('localhost:9000') - bucket_iter = client.list_objects('bucket', recursive=True) - buckets = [] - for bucket in bucket_iter: - mock_server.mock_add_request( - MockResponse( - "GET", - "https://localhost:9000/bucket/" - "?delimiter=&marker=marker&max-keys=1000&prefix=", - {"User-Agent": _DEFAULT_USER_AGENT}, - 200, - content=mock_data2, - ), - ) - buckets.append(bucket) + objects_iter = client.list_objects('bucket') + objects = [] + for obj in objects_iter: + objects.append(obj) - eq_(4, len(buckets)) + eq_(2, len(objects)) diff --git a/tests/unit/list_objects_v1_test.py b/tests/unit/list_objects_v1_test.py new file mode 100644 index 000000000..05bbcaa1f --- /dev/null +++ b/tests/unit/list_objects_v1_test.py @@ -0,0 +1,222 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, +# (C) 2015-2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import TestCase + +from nose.tools import eq_, timed + +import mock +from minio import Minio +from minio.api import _DEFAULT_USER_AGENT + +from .minio_mocks import MockConnection, MockResponse + + +class ListObjectsV1Test(TestCase): + @mock.patch('urllib3.PoolManager') + def test_empty_list_objects_works(self, mock_connection): + mock_data = ''' + + bucket + + + false + 1000 + +''' + mock_server = MockConnection() + mock_connection.return_value = mock_server + mock_server.mock_add_request( + MockResponse( + "GET", + "https://localhost:9000/bucket/" + "?delimiter=&max-keys=1000&prefix=", + {"User-Agent": _DEFAULT_USER_AGENT}, + 200, + content=mock_data, + ), + ) + client = Minio('localhost:9000') + bucket_iter = client.list_objects( + 'bucket', recursive=True, use_api_v1=True, + ) + buckets = [] + for bucket in bucket_iter: + buckets.append(bucket) + eq_(0, len(buckets)) + + @timed(1) + @mock.patch('urllib3.PoolManager') + def test_list_objects_works(self, mock_connection): + mock_data = ''' + + bucket + + + 1000 + + false + + key1 + 2015-05-05T02:21:15.716Z + 5eb63bbbe01eeed093cb22bb8f5acdc3 + 11 + STANDARD + + minio + minio + + + + key2 + 2015-05-05T20:36:17.498Z + 2a60eaffa7a82804bdc682ce1df6c2d4 + 1661 + STANDARD + + minio + minio + + +''' + mock_server = MockConnection() + mock_connection.return_value = mock_server + mock_server.mock_add_request( + MockResponse( + "GET", + "https://localhost:9000/bucket/" + "?delimiter=%2F&max-keys=1000&prefix=", + {"User-Agent": _DEFAULT_USER_AGENT}, + 200, + content=mock_data, + ), + ) + client = Minio('localhost:9000') + bucket_iter = client.list_objects('bucket', use_api_v1=True) + buckets = [] + for bucket in bucket_iter: + # cause an xml exception and fail if we try retrieving again + mock_server.mock_add_request( + MockResponse( + "GET", + "https://localhost:9000/bucket/" + "?delimiter=%2F&max-keys=1000&prefix=", + {"User-Agent": _DEFAULT_USER_AGENT}, + 200, + content="", + ), + ) + buckets.append(bucket) + + eq_(2, len(buckets)) + + @timed(1) + @mock.patch('urllib3.PoolManager') + def test_list_objects_works_well(self, mock_connection): + mock_data1 = ''' + + bucket + + + marker + 1000 + + true + + key1 + 2015-05-05T02:21:15.716Z + 5eb63bbbe01eeed093cb22bb8f5acdc3 + 11 + STANDARD + + minio + minio + + + + key2 + 2015-05-05T20:36:17.498Z + 2a60eaffa7a82804bdc682ce1df6c2d4 + 1661 + STANDARD + + minio + minio + + +''' + mock_data2 = ''' + + bucket + + + 1000 + + false + + key3 + 2015-05-05T02:21:15.716Z + 5eb63bbbe01eeed093cb22bb8f5acdc3 + 11 + STANDARD + + minio + minio + + + + key4 + 2015-05-05T20:36:17.498Z + 2a60eaffa7a82804bdc682ce1df6c2d4 + 1661 + STANDARD + + minio + minio + + +''' + mock_server = MockConnection() + mock_connection.return_value = mock_server + mock_server.mock_add_request( + MockResponse( + "GET", + "https://localhost:9000/bucket/" + "?delimiter=&max-keys=1000&prefix=", + {"User-Agent": _DEFAULT_USER_AGENT}, + 200, + content=mock_data1, + ), + ) + client = Minio('localhost:9000') + bucket_iter = client.list_objects( + 'bucket', recursive=True, use_api_v1=True, + ) + buckets = [] + for bucket in bucket_iter: + mock_server.mock_add_request( + MockResponse( + "GET", + "https://localhost:9000/bucket/" + "?delimiter=&marker=marker&max-keys=1000&prefix=", + {"User-Agent": _DEFAULT_USER_AGENT}, + 200, + content=mock_data2, + ), + ) + buckets.append(bucket) + + eq_(4, len(buckets)) diff --git a/tests/unit/list_objects_v2_test.py b/tests/unit/list_objects_v2_test.py deleted file mode 100644 index 18665e146..000000000 --- a/tests/unit/list_objects_v2_test.py +++ /dev/null @@ -1,102 +0,0 @@ -# -*- coding: utf-8 -*- -# MinIO Python Library for Amazon S3 Compatible Cloud Storage, -# (C) 2015 MinIO, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from unittest import TestCase - -from nose.tools import eq_, timed - -import mock -from minio import Minio -from minio.api import _DEFAULT_USER_AGENT - -from .minio_mocks import MockConnection, MockResponse - - -class ListObjectsV2Test(TestCase): - @mock.patch('urllib3.PoolManager') - def test_empty_list_objects_works(self, mock_connection): - mock_data = ''' - - bucket - - 0 - 1000 - - false -''' - mock_server = MockConnection() - mock_connection.return_value = mock_server - mock_server.mock_add_request( - MockResponse( - "GET", - "https://localhost:9000/bucket/?delimiter=&list-type=2" - "&max-keys=1000&prefix=", - {"User-Agent": _DEFAULT_USER_AGENT}, - 200, - content=mock_data, - ), - ) - client = Minio('localhost:9000') - object_iter = client.list_objects_v2('bucket', recursive=True) - objects = [] - for obj in object_iter: - objects.append(obj) - eq_(0, len(objects)) - - @timed(1) - @mock.patch('urllib3.PoolManager') - def test_list_objects_works(self, mock_connection): - mock_data = ''' - - bucket - - 2 - 1000 - false - - 6/f/9/6f9898076bb08572403f95dbb86c5b9c85e1e1b3 - 2016-11-27T07:55:53.000Z - "5d5512301b6b6e247b8aec334b2cf7ea" - 493 - REDUCED_REDUNDANCY - - - b/d/7/bd7f6410cced55228902d881c2954ebc826d7464 - 2016-11-27T07:10:27.000Z - "f00483d523ffc8b7f2883ae896769d85" - 493 - REDUCED_REDUNDANCY - -''' - mock_server = MockConnection() - mock_connection.return_value = mock_server - mock_server.mock_add_request( - MockResponse( - "GET", - "https://localhost:9000/bucket/?delimiter=%2F&list-type=2" - "&max-keys=1000&prefix=", - {"User-Agent": _DEFAULT_USER_AGENT}, - 200, - content=mock_data, - ), - ) - client = Minio('localhost:9000') - objects_iter = client.list_objects_v2('bucket') - objects = [] - for obj in objects_iter: - objects.append(obj) - - eq_(2, len(objects))