diff --git a/minio/__init__.py b/minio/__init__.py index b936008a1..76002bf4c 100644 --- a/minio/__init__.py +++ b/minio/__init__.py @@ -29,7 +29,7 @@ __title__ = 'minio-py' __author__ = 'MinIO, Inc.' -__version__ = '6.0.1' +__version__ = '7.0.0' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2015, 2016, 2017, 2018, 2019, 2020 MinIO, Inc.' diff --git a/minio/api.py b/minio/api.py index bd75970a0..7465a7a27 100644 --- a/minio/api.py +++ b/minio/api.py @@ -1039,53 +1039,8 @@ def put_object(self, bucket_name, object_name, data, length, progress=progress) def list_objects(self, bucket_name, prefix=None, recursive=False, - include_version=False): - """ - Lists object information of a bucket using S3 API version 1, optionally - for prefix recursively. - - :param bucket_name: Name of the bucket. - :param prefix: Object name starts with prefix. - :param recursive: List recursively than directory structure emulation. - :param include_version: Flag to control whether include object - versions. - :return: An iterator contains object information. - - Example:: - # List objects information. - objects = minio.list_objects('foo') - for object in objects: - print(object) - - # List objects information those names starts with 'hello/'. - objects = minio.list_objects('foo', prefix='hello/') - for object in objects: - print(object) - - # List objects information recursively. - objects = minio.list_objects('foo', recursive=True) - for object in objects: - print(object) - - # List objects information recursively those names starts with - # 'hello/'. - objects = minio.list_objects( - 'foo', prefix='hello/', recursive=True, - ) - for object in objects: - print(object) - """ - return self._list_objects( - bucket_name, - delimiter=None if recursive else "/", - prefix=prefix, - use_version1=True, - include_version=include_version, - ) - - def list_objects_v2(self, bucket_name, prefix=None, recursive=False, - start_after=None, include_user_meta=False, - include_version=False): + start_after=None, include_user_meta=False, + include_version=False, use_api_v1=False): """ Lists object information of a bucket using S3 API version 2, optionally for prefix recursively. @@ -1098,27 +1053,28 @@ def list_objects_v2(self, bucket_name, prefix=None, recursive=False, user metadata. :param include_version: Flag to control whether include object versions. + :param use_api_v1: Flag to control to use ListObjectV1 S3 API or not. :return: An iterator contains object information. Example:: # List objects information. - objects = minio.list_objects_v2('foo') + objects = minio.list_objects('foo') for object in objects: print(object) # List objects information those names starts with 'hello/'. - objects = minio.list_objects_v2('foo', prefix='hello/') + objects = minio.list_objects('foo', prefix='hello/') for object in objects: print(object) # List objects information recursively. - objects = minio.list_objects_v2('foo', recursive=True) + objects = minio.list_objects('foo', recursive=True) for object in objects: print(object) # List objects information recursively those names starts with # 'hello/'. - objects = minio.list_objects_v2( + objects = minio.list_objects( 'foo', prefix='hello/', recursive=True, ) for object in objects: @@ -1126,7 +1082,7 @@ def list_objects_v2(self, bucket_name, prefix=None, recursive=False, # List objects information recursively after object name # 'hello/world/1'. - objects = minio.list_objects_v2( + objects = minio.list_objects( 'foo', recursive=True, start_after='hello/world/1', ) for object in objects: @@ -1138,6 +1094,7 @@ def list_objects_v2(self, bucket_name, prefix=None, recursive=False, include_user_meta=include_user_meta, prefix=prefix, start_after=start_after, + use_api_v1=use_api_v1, include_version=include_version, ) @@ -2151,7 +2108,7 @@ def _list_objects( # pylint: disable=too-many-arguments,too-many-branches prefix=None, # all start_after=None, # all: v1:marker, versioned:key_marker version_id_marker=None, # versioned - use_version1=False, + use_api_v1=False, include_version=False, ): """ @@ -2172,10 +2129,10 @@ def _list_objects( # pylint: disable=too-many-arguments,too-many-branches query = {} if include_version: query["versions"] = "" - elif not use_version1: + elif not use_api_v1: query["list-type"] = "2" - if not include_version and not use_version1: + if not include_version and not use_api_v1: if continuation_token: query["continuation-token"] = continuation_token if fetch_owner: @@ -2190,7 +2147,7 @@ def _list_objects( # pylint: disable=too-many-arguments,too-many-branches if start_after: if include_version: query["key-marker"] = start_after - elif use_version1: + elif use_api_v1: query["marker"] = start_after else: query["start-after"] = start_after @@ -2207,7 +2164,7 @@ def _list_objects( # pylint: disable=too-many-arguments,too-many-branches objects, is_truncated, start_after, version_id_marker = ( parse_list_object_versions(response.data, bucket_name) ) - elif use_version1: + elif use_api_v1: objects, is_truncated, start_after = parse_list_objects( response.data, bucket_name, diff --git a/tests/functional/tests.py b/tests/functional/tests.py index 7b732234c..2390003fe 100644 --- a/tests/functional/tests.py +++ b/tests/functional/tests.py @@ -1064,7 +1064,7 @@ def test_get_partial_object(log_entry, sse=None): _CLIENT.remove_bucket(bucket_name) -def _test_list_objects(log_entry, version2=False, version_check=False): +def _test_list_objects(log_entry, use_api_v1=False, version_check=False): """Test list_objects().""" # Get a unique bucket_name and object_name @@ -1092,14 +1092,10 @@ def _test_list_objects(log_entry, version2=False, version_check=False): bucket_name, object_name + "-2", LimitedRandomReader(size), size, ) # List all object paths in bucket. - if version2: - objects = _CLIENT.list_objects_v2( - bucket_name, '', is_recursive, include_version=version_check, - ) - else: - objects = _CLIENT.list_objects( - bucket_name, '', is_recursive, include_version=version_check, - ) + objects = _CLIENT.list_objects( + bucket_name, '', is_recursive, include_version=version_check, + use_api_v1=use_api_v1, + ) for obj in objects: _ = (obj.bucket_name, obj.object_name, obj.last_modified, obj.etag, obj.size, obj.content_type) @@ -1119,14 +1115,14 @@ def _test_list_objects(log_entry, version2=False, version_check=False): _CLIENT.remove_bucket(bucket_name) -def test_list_objects(log_entry): +def test_list_objects_v1(log_entry): """Test list_objects().""" - _test_list_objects(log_entry) + _test_list_objects(log_entry, use_api_v1=True) -def test_list_object_versions(log_entry): +def test_list_object_v1_versions(log_entry): """Test list_objects().""" - _test_list_objects(log_entry, version_check=True) + _test_list_objects(log_entry, use_api_v1=True, version_check=True) def _test_list_objects_api(bucket_name, expected_no, *argv): @@ -1266,14 +1262,14 @@ def test_list_objects_with_1001_files( # pylint: disable=invalid-name _CLIENT.remove_bucket(bucket_name) -def test_list_objects_v2(log_entry): - """Test list_objects_v2().""" - _test_list_objects(log_entry, version2=True) +def test_list_objects(log_entry): + """Test list_objects().""" + _test_list_objects(log_entry) -def test_list_object_versions_v2(log_entry): - """Test list_objects_v2() of versioned object.""" - _test_list_objects(log_entry, version2=True, version_check=True) +def test_list_object_versions(log_entry): + """Test list_objects() of versioned object.""" + _test_list_objects(log_entry, version_check=True) def _create_upload_ids(bucket_name, object_name, count): @@ -1982,13 +1978,13 @@ def main(): test_fget_object_version: {"sse": ssec} if ssec else None, test_get_object_with_default_length: None, test_get_partial_object: {"sse": ssec} if ssec else None, - test_list_objects: None, - test_list_object_versions: None, + test_list_objects_v1: None, + test_list_object_v1_versions: None, test_list_objects_with_prefix: None, test_list_objects_with_1001_files: None, test_remove_incomplete_upload: None, - test_list_objects_v2: None, - test_list_object_versions_v2: None, + test_list_objects: None, + test_list_object_versions: None, test_presigned_get_object_default_expiry: None, test_presigned_get_object_expiry: None, test_presigned_get_object_response_headers: None, diff --git a/tests/unit/list_objects_test.py b/tests/unit/list_objects_test.py index a807bf729..eb4f992fc 100644 --- a/tests/unit/list_objects_test.py +++ b/tests/unit/list_objects_test.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # MinIO Python Library for Amazon S3 Compatible Cloud Storage, -# (C) 2015, 2016 MinIO, Inc. +# (C) 2015-2020 MinIO, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -28,66 +28,57 @@ class ListObjectsTest(TestCase): @mock.patch('urllib3.PoolManager') def test_empty_list_objects_works(self, mock_connection): - mock_data = ''' + mock_data = ''' bucket - - - false + + 0 1000 - + + false ''' mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request( MockResponse( "GET", - "https://localhost:9000/bucket/" - "?delimiter=&max-keys=1000&prefix=", + "https://localhost:9000/bucket/?delimiter=&list-type=2" + "&max-keys=1000&prefix=", {"User-Agent": _DEFAULT_USER_AGENT}, 200, content=mock_data, ), ) client = Minio('localhost:9000') - bucket_iter = client.list_objects('bucket', recursive=True) - buckets = [] - for bucket in bucket_iter: - buckets.append(bucket) - eq_(0, len(buckets)) + object_iter = client.list_objects('bucket', recursive=True) + objects = [] + for obj in object_iter: + objects.append(obj) + eq_(0, len(objects)) @timed(1) @mock.patch('urllib3.PoolManager') def test_list_objects_works(self, mock_connection): - mock_data = ''' + mock_data = ''' bucket - - + + 2 1000 - false - key1 - 2015-05-05T02:21:15.716Z - 5eb63bbbe01eeed093cb22bb8f5acdc3 - 11 - STANDARD - - minio - minio - + 6/f/9/6f9898076bb08572403f95dbb86c5b9c85e1e1b3 + 2016-11-27T07:55:53.000Z + "5d5512301b6b6e247b8aec334b2cf7ea" + 493 + REDUCED_REDUNDANCY - key2 - 2015-05-05T20:36:17.498Z - 2a60eaffa7a82804bdc682ce1df6c2d4 - 1661 - STANDARD - - minio - minio - + b/d/7/bd7f6410cced55228902d881c2954ebc826d7464 + 2016-11-27T07:10:27.000Z + "f00483d523ffc8b7f2883ae896769d85" + 493 + REDUCED_REDUNDANCY ''' mock_server = MockConnection() @@ -95,124 +86,17 @@ def test_list_objects_works(self, mock_connection): mock_server.mock_add_request( MockResponse( "GET", - "https://localhost:9000/bucket/" - "?delimiter=%2F&max-keys=1000&prefix=", + "https://localhost:9000/bucket/?delimiter=%2F&list-type=2" + "&max-keys=1000&prefix=", {"User-Agent": _DEFAULT_USER_AGENT}, 200, content=mock_data, ), ) client = Minio('localhost:9000') - bucket_iter = client.list_objects('bucket') - buckets = [] - for bucket in bucket_iter: - # cause an xml exception and fail if we try retrieving again - mock_server.mock_add_request( - MockResponse( - "GET", - "https://localhost:9000/bucket/" - "?delimiter=%2F&max-keys=1000&prefix=", - {"User-Agent": _DEFAULT_USER_AGENT}, - 200, - content="", - ), - ) - buckets.append(bucket) - - eq_(2, len(buckets)) - - @timed(1) - @mock.patch('urllib3.PoolManager') - def test_list_objects_works_well(self, mock_connection): - mock_data1 = ''' - - bucket - - - marker - 1000 - - true - - key1 - 2015-05-05T02:21:15.716Z - 5eb63bbbe01eeed093cb22bb8f5acdc3 - 11 - STANDARD - - minio - minio - - - - key2 - 2015-05-05T20:36:17.498Z - 2a60eaffa7a82804bdc682ce1df6c2d4 - 1661 - STANDARD - - minio - minio - - -''' - mock_data2 = ''' - - bucket - - - 1000 - - false - - key3 - 2015-05-05T02:21:15.716Z - 5eb63bbbe01eeed093cb22bb8f5acdc3 - 11 - STANDARD - - minio - minio - - - - key4 - 2015-05-05T20:36:17.498Z - 2a60eaffa7a82804bdc682ce1df6c2d4 - 1661 - STANDARD - - minio - minio - - -''' - mock_server = MockConnection() - mock_connection.return_value = mock_server - mock_server.mock_add_request( - MockResponse( - "GET", - "https://localhost:9000/bucket/" - "?delimiter=&max-keys=1000&prefix=", - {"User-Agent": _DEFAULT_USER_AGENT}, - 200, - content=mock_data1, - ), - ) - client = Minio('localhost:9000') - bucket_iter = client.list_objects('bucket', recursive=True) - buckets = [] - for bucket in bucket_iter: - mock_server.mock_add_request( - MockResponse( - "GET", - "https://localhost:9000/bucket/" - "?delimiter=&marker=marker&max-keys=1000&prefix=", - {"User-Agent": _DEFAULT_USER_AGENT}, - 200, - content=mock_data2, - ), - ) - buckets.append(bucket) + objects_iter = client.list_objects('bucket') + objects = [] + for obj in objects_iter: + objects.append(obj) - eq_(4, len(buckets)) + eq_(2, len(objects)) diff --git a/tests/unit/list_objects_v1_test.py b/tests/unit/list_objects_v1_test.py new file mode 100644 index 000000000..05bbcaa1f --- /dev/null +++ b/tests/unit/list_objects_v1_test.py @@ -0,0 +1,222 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, +# (C) 2015-2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import TestCase + +from nose.tools import eq_, timed + +import mock +from minio import Minio +from minio.api import _DEFAULT_USER_AGENT + +from .minio_mocks import MockConnection, MockResponse + + +class ListObjectsV1Test(TestCase): + @mock.patch('urllib3.PoolManager') + def test_empty_list_objects_works(self, mock_connection): + mock_data = ''' + + bucket + + + false + 1000 + +''' + mock_server = MockConnection() + mock_connection.return_value = mock_server + mock_server.mock_add_request( + MockResponse( + "GET", + "https://localhost:9000/bucket/" + "?delimiter=&max-keys=1000&prefix=", + {"User-Agent": _DEFAULT_USER_AGENT}, + 200, + content=mock_data, + ), + ) + client = Minio('localhost:9000') + bucket_iter = client.list_objects( + 'bucket', recursive=True, use_api_v1=True, + ) + buckets = [] + for bucket in bucket_iter: + buckets.append(bucket) + eq_(0, len(buckets)) + + @timed(1) + @mock.patch('urllib3.PoolManager') + def test_list_objects_works(self, mock_connection): + mock_data = ''' + + bucket + + + 1000 + + false + + key1 + 2015-05-05T02:21:15.716Z + 5eb63bbbe01eeed093cb22bb8f5acdc3 + 11 + STANDARD + + minio + minio + + + + key2 + 2015-05-05T20:36:17.498Z + 2a60eaffa7a82804bdc682ce1df6c2d4 + 1661 + STANDARD + + minio + minio + + +''' + mock_server = MockConnection() + mock_connection.return_value = mock_server + mock_server.mock_add_request( + MockResponse( + "GET", + "https://localhost:9000/bucket/" + "?delimiter=%2F&max-keys=1000&prefix=", + {"User-Agent": _DEFAULT_USER_AGENT}, + 200, + content=mock_data, + ), + ) + client = Minio('localhost:9000') + bucket_iter = client.list_objects('bucket', use_api_v1=True) + buckets = [] + for bucket in bucket_iter: + # cause an xml exception and fail if we try retrieving again + mock_server.mock_add_request( + MockResponse( + "GET", + "https://localhost:9000/bucket/" + "?delimiter=%2F&max-keys=1000&prefix=", + {"User-Agent": _DEFAULT_USER_AGENT}, + 200, + content="", + ), + ) + buckets.append(bucket) + + eq_(2, len(buckets)) + + @timed(1) + @mock.patch('urllib3.PoolManager') + def test_list_objects_works_well(self, mock_connection): + mock_data1 = ''' + + bucket + + + marker + 1000 + + true + + key1 + 2015-05-05T02:21:15.716Z + 5eb63bbbe01eeed093cb22bb8f5acdc3 + 11 + STANDARD + + minio + minio + + + + key2 + 2015-05-05T20:36:17.498Z + 2a60eaffa7a82804bdc682ce1df6c2d4 + 1661 + STANDARD + + minio + minio + + +''' + mock_data2 = ''' + + bucket + + + 1000 + + false + + key3 + 2015-05-05T02:21:15.716Z + 5eb63bbbe01eeed093cb22bb8f5acdc3 + 11 + STANDARD + + minio + minio + + + + key4 + 2015-05-05T20:36:17.498Z + 2a60eaffa7a82804bdc682ce1df6c2d4 + 1661 + STANDARD + + minio + minio + + +''' + mock_server = MockConnection() + mock_connection.return_value = mock_server + mock_server.mock_add_request( + MockResponse( + "GET", + "https://localhost:9000/bucket/" + "?delimiter=&max-keys=1000&prefix=", + {"User-Agent": _DEFAULT_USER_AGENT}, + 200, + content=mock_data1, + ), + ) + client = Minio('localhost:9000') + bucket_iter = client.list_objects( + 'bucket', recursive=True, use_api_v1=True, + ) + buckets = [] + for bucket in bucket_iter: + mock_server.mock_add_request( + MockResponse( + "GET", + "https://localhost:9000/bucket/" + "?delimiter=&marker=marker&max-keys=1000&prefix=", + {"User-Agent": _DEFAULT_USER_AGENT}, + 200, + content=mock_data2, + ), + ) + buckets.append(bucket) + + eq_(4, len(buckets)) diff --git a/tests/unit/list_objects_v2_test.py b/tests/unit/list_objects_v2_test.py deleted file mode 100644 index 18665e146..000000000 --- a/tests/unit/list_objects_v2_test.py +++ /dev/null @@ -1,102 +0,0 @@ -# -*- coding: utf-8 -*- -# MinIO Python Library for Amazon S3 Compatible Cloud Storage, -# (C) 2015 MinIO, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from unittest import TestCase - -from nose.tools import eq_, timed - -import mock -from minio import Minio -from minio.api import _DEFAULT_USER_AGENT - -from .minio_mocks import MockConnection, MockResponse - - -class ListObjectsV2Test(TestCase): - @mock.patch('urllib3.PoolManager') - def test_empty_list_objects_works(self, mock_connection): - mock_data = ''' - - bucket - - 0 - 1000 - - false -''' - mock_server = MockConnection() - mock_connection.return_value = mock_server - mock_server.mock_add_request( - MockResponse( - "GET", - "https://localhost:9000/bucket/?delimiter=&list-type=2" - "&max-keys=1000&prefix=", - {"User-Agent": _DEFAULT_USER_AGENT}, - 200, - content=mock_data, - ), - ) - client = Minio('localhost:9000') - object_iter = client.list_objects_v2('bucket', recursive=True) - objects = [] - for obj in object_iter: - objects.append(obj) - eq_(0, len(objects)) - - @timed(1) - @mock.patch('urllib3.PoolManager') - def test_list_objects_works(self, mock_connection): - mock_data = ''' - - bucket - - 2 - 1000 - false - - 6/f/9/6f9898076bb08572403f95dbb86c5b9c85e1e1b3 - 2016-11-27T07:55:53.000Z - "5d5512301b6b6e247b8aec334b2cf7ea" - 493 - REDUCED_REDUNDANCY - - - b/d/7/bd7f6410cced55228902d881c2954ebc826d7464 - 2016-11-27T07:10:27.000Z - "f00483d523ffc8b7f2883ae896769d85" - 493 - REDUCED_REDUNDANCY - -''' - mock_server = MockConnection() - mock_connection.return_value = mock_server - mock_server.mock_add_request( - MockResponse( - "GET", - "https://localhost:9000/bucket/?delimiter=%2F&list-type=2" - "&max-keys=1000&prefix=", - {"User-Agent": _DEFAULT_USER_AGENT}, - 200, - content=mock_data, - ), - ) - client = Minio('localhost:9000') - objects_iter = client.list_objects_v2('bucket') - objects = [] - for obj in objects_iter: - objects.append(obj) - - eq_(2, len(objects))