From 36000160a9e1b8517a6554b7675f6f7ee2d079e9 Mon Sep 17 00:00:00 2001 From: pcotte Date: Wed, 17 Apr 2024 18:10:37 +0200 Subject: [PATCH 01/26] feat: support bucket versioning --- pytest_minio_mock/plugin.py | 33 +++++++++++++++++++++++++++++++++ tests/test_minio_mock.py | 15 +++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/pytest_minio_mock/plugin.py b/pytest_minio_mock/plugin.py index c9bddb4..bb3dad4 100644 --- a/pytest_minio_mock/plugin.py +++ b/pytest_minio_mock/plugin.py @@ -31,6 +31,7 @@ from minio import Minio from minio.datatypes import Object from minio.error import S3Error +from minio.versioningconfig import VersioningConfig from urllib3.connection import HTTPConnection from urllib3.response import HTTPResponse @@ -610,6 +611,38 @@ def make_bucket(self, bucket_name, location=None, object_lock=False): } return True + def set_bucket_versioning(self, bucket_name: str, config: VersioningConfig): + self._health_check() + if not self.bucket_exists(bucket_name): + raise S3Error( + message="bucket does not exist", + resource=bucket_name, + request_id=None, + host_id=None, + response="mocked_response", + code=404, + bucket_name=bucket_name, + object_name=None, + ) + if not isinstance(config, VersioningConfig): + raise ValueError("config must be VersioningConfig type") + self.buckets[bucket_name]["versioning"] = config + + def get_bucket_versioning(self, bucket_name: str) -> VersioningConfig: + self._health_check() + if not self.bucket_exists(bucket_name): + raise S3Error( + message="bucket does not exist", + resource=bucket_name, + request_id=None, + host_id=None, + response="mocked_response", + code=404, + bucket_name=bucket_name, + object_name=None, + ) + return self.buckets[bucket_name].get("versioning", VersioningConfig("Suspended")) + def list_objects(self, bucket_name, prefix="", recursive=False, start_after=""): """ Lists objects in a bucket with the specified prefix and conditions. diff --git a/tests/test_minio_mock.py b/tests/test_minio_mock.py index 86844f5..5a00784 100644 --- a/tests/test_minio_mock.py +++ b/tests/test_minio_mock.py @@ -4,6 +4,8 @@ import validators from minio import Minio from minio.error import S3Error +from minio.commonconfig import ENABLED +from minio.versioningconfig import VersioningConfig @pytest.mark.UNIT @@ -63,6 +65,19 @@ def test_bucket_exists(minio_mock): assert client.bucket_exists(bucket_name), "Bucket should exist" +@pytest.mark.UNIT +@pytest.mark.API +def test_bucket_versioning(minio_mock): + bucket_name = "existing-bucket" + client = Minio("http://local.host:9000") + client.make_bucket(bucket_name) + assert client.get_bucket_versioning(bucket_name).status == VersioningConfig("Suspended").status + client.set_bucket_versioning(bucket_name, VersioningConfig(ENABLED)) + assert client.get_bucket_versioning(bucket_name).status == VersioningConfig(ENABLED).status + client.set_bucket_versioning(bucket_name, VersioningConfig("Suspended")) + assert client.get_bucket_versioning(bucket_name).status == VersioningConfig("Suspended").status + + @pytest.mark.UNIT @pytest.mark.API def test_get_presigned_url(minio_mock): From 4b04e5d376a2593cd96d6ffb894e30eeb0ee269f Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sat, 20 Apr 2024 13:32:36 +0200 Subject: [PATCH 02/26] reverted breaking changes --- tests/conftest.py | 8 -------- tests/test_minio_mock.py | 40 +++++++++++++++++++++------------------- 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index c64a233..ef57091 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,2 @@ -import pytest from pytest_minio_mock.plugin import minio_mock from pytest_minio_mock.plugin import minio_mock_servers -from minio import Minio - - -@pytest.fixture(autouse=True) -def _clean(minio_mock): - client = Minio("http://local.host:9000") - client.buckets = {} diff --git a/tests/test_minio_mock.py b/tests/test_minio_mock.py index 02c7fe7..151235d 100644 --- a/tests/test_minio_mock.py +++ b/tests/test_minio_mock.py @@ -3,14 +3,14 @@ import pytest import validators from minio import Minio -from minio.error import S3Error from minio.commonconfig import ENABLED +from minio.error import S3Error from minio.versioningconfig import VersioningConfig @pytest.mark.UNIT @pytest.mark.API -def test_make_bucket(): +def test_make_bucket(minio_mock): bucket_name = "test-bucket" client = Minio("http://local.host:9000") client.make_bucket(bucket_name) @@ -19,7 +19,7 @@ def test_make_bucket(): @pytest.mark.UNIT @pytest.mark.API -def test_adding_and_removing_objects(): +def test_adding_and_removing_objects(minio_mock): bucket_name = "test-bucket" object_name = "test-object" file_path = "tests/fixtures/maya.jpeg" @@ -38,7 +38,7 @@ def test_adding_and_removing_objects(): @pytest.mark.UNIT @pytest.mark.API -def test_versioned_objects(): +def test_versioned_objects(minio_mock): bucket_name = "test-bucket" object_name = "test-object" file_path = "tests/fixtures/maya.jpeg" @@ -81,7 +81,7 @@ def test_versioned_objects(): @pytest.mark.UNIT @pytest.mark.API -def test_versioned_objects_after_upload(): +def test_versioned_objects_after_upload(minio_mock): bucket_name = "test-bucket" object_name = "test-object" file_path = "tests/fixtures/maya.jpeg" @@ -116,7 +116,7 @@ def test_versioned_objects_after_upload(): @pytest.mark.UNIT @pytest.mark.API @pytest.mark.parametrize("versioned", (True, False)) -def test_file_download(versioned): +def test_file_download(minio_mock, versioned): bucket_name = "test-bucket" object_name = "test-object" file_content = b"Test file content" @@ -149,7 +149,7 @@ def test_file_download(versioned): @pytest.mark.UNIT @pytest.mark.API -def test_bucket_exists(): +def test_bucket_exists(minio_mock): bucket_name = "existing-bucket" client = Minio("http://local.host:9000") client.make_bucket(bucket_name) @@ -158,7 +158,7 @@ def test_bucket_exists(): @pytest.mark.UNIT @pytest.mark.API -def test_bucket_versioning(): +def test_bucket_versioning(minio_mock): bucket_name = "existing-bucket" client = Minio("http://local.host:9000") client.make_bucket(bucket_name) @@ -172,7 +172,7 @@ def test_bucket_versioning(): @pytest.mark.UNIT @pytest.mark.API @pytest.mark.parametrize("versioned", (True, False)) -def test_get_presigned_url(versioned): +def test_get_presigned_url(minio_mock, versioned): bucket_name = "test-bucket" object_name = "test-object" file_path = "tests/fixtures/maya.jpeg" @@ -184,10 +184,10 @@ def test_get_presigned_url(versioned): client.set_bucket_versioning(bucket_name, VersioningConfig(ENABLED)) client.fput_object(bucket_name, object_name, file_path) if versioned: - version = list(client.list_objects(bucket_name, object_name, include_version=True))[-1].version_id - url = client.get_presigned_url( - "GET", bucket_name, object_name, version_id=version - ) + version = list( + client.list_objects(bucket_name, object_name, include_version=True) + )[-1].version_id + url = client.get_presigned_url("GET", bucket_name, object_name, version_id=version) assert validators.url(url) if version: assert url.endswith(f"?versionId={version}") @@ -195,7 +195,7 @@ def test_get_presigned_url(versioned): @pytest.mark.UNIT @pytest.mark.API -def test_presigned_put_url(): +def test_presigned_put_url(minio_mock): bucket_name = "test-bucket" object_name = "test-object" file_path = "tests/fixtures/maya.jpeg" @@ -209,7 +209,7 @@ def test_presigned_put_url(): @pytest.mark.UNIT @pytest.mark.API -def test_presigned_get_url(): +def test_presigned_get_url(minio_mock): bucket_name = "test-bucket" object_name = "test-object" file_path = "tests/fixtures/maya.jpeg" @@ -223,7 +223,7 @@ def test_presigned_get_url(): @pytest.mark.UNIT @pytest.mark.API -def test_list_buckets(): +def test_list_buckets(minio_mock): client = Minio("http://local.host:9000") buckets = client.list_buckets() n = len(buckets) @@ -236,7 +236,7 @@ def test_list_buckets(): @pytest.mark.REGRESSION @pytest.mark.UNIT @pytest.mark.API -def test_list_objects(): +def test_list_objects(minio_mock): client = Minio("http://local.host:9000") with pytest.raises(S3Error): @@ -253,7 +253,9 @@ def test_list_objects(): client.put_object(bucket_name, "object4", data=b"object4 data", length=11) # Test recursive listing - objects_recursive = list(client.list_objects(bucket_name, prefix="a/", recursive=True)) + objects_recursive = list( + client.list_objects(bucket_name, prefix="a/", recursive=True) + ) assert len(objects_recursive) == 3, "Expected 3 objects under 'a/' with recursion" # Check that all expected paths are returned assert set(obj.object_name for obj in objects_recursive) == { @@ -280,7 +282,7 @@ def test_list_objects(): @pytest.mark.REGRESSION -def test_connecting_to_the_same_endpoint(): +def test_connecting_to_the_same_endpoint(minio_mock): client_1 = Minio("http://local.host:9000") client_1_buckets = ["bucket-1", "bucket-2", "bucket-3"] for bucket in client_1_buckets: From 9af5c642ad996c481e23891aa5e58a05f19ffc7b Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sat, 20 Apr 2024 14:18:11 +0200 Subject: [PATCH 03/26] added the MockMinioBucket class --- pytest_minio_mock/plugin.py | 86 ++++++++++++++++++++++++++----------- tests/test_minio_mock.py | 30 +++++++++++++ 2 files changed, 92 insertions(+), 24 deletions(-) diff --git a/pytest_minio_mock/plugin.py b/pytest_minio_mock/plugin.py index ccf4e1a..9c3b149 100644 --- a/pytest_minio_mock/plugin.py +++ b/pytest_minio_mock/plugin.py @@ -29,10 +29,12 @@ import pytest import validators from minio import Minio +from minio.commonconfig import ENABLED from minio.datatypes import Object from minio.error import S3Error -from minio.commonconfig import ENABLED -from minio.versioningconfig import VersioningConfig, OFF, SUSPENDED +from minio.versioningconfig import OFF +from minio.versioningconfig import SUSPENDED +from minio.versioningconfig import VersioningConfig from urllib3.connection import HTTPConnection from urllib3.response import HTTPResponse @@ -53,7 +55,7 @@ class MockMinioObject: versioned bucket """ - def __init__(self, object_name, data, version_id=None, is_delete_marker=False): + def __init__(self, object_name, data, version_id, is_delete_marker): """ Initialize the MockMinioObject with a name and data. @@ -95,6 +97,33 @@ def object_name(self): return self._object_name +class MockMinioBucket: + def __init__(self, bucket_name: str, versioning: VersioningConfig): + self._bucket_name = bucket_name + self._objects = {} + self._versioning = versioning + + @property + def bucket_name(self): + """Get the name of the bucket.""" + return self._bucket_name + + @property + def objects(self): + """Get the objects stored in the bucket.""" + return self._objects + + @property + def versioning(self): + """Get the the config of versioning of the bucket.""" + return self._versioning + + @versioning.setter + def versioning(self, versioning: VersioningConfig): + """Set the versioning config of the bucket.""" + self._versioning = versioning + + class MockMinioServer: """ Represents a mock Minio server. @@ -352,7 +381,7 @@ def get_object( object_name=object_name, ) try: - the_object = self.buckets[bucket_name]["objects"][object_name] + the_object = self.buckets[bucket_name].objects[object_name] except KeyError: raise nosuchkey @@ -547,9 +576,7 @@ def put_object( # If status is enabled, create a new version UUID version = str(uuid4()) - obj = MockMinioObject( - object_name=object_name, data=data, version_id=version - ) + obj = MockMinioObject(object_name=object_name, data=data, version_id=version) # If versioning is OFF, there can only be one version (None) of an object if self.get_bucket_versioning(bucket_name).status == OFF: self.buckets[bucket_name]["objects"][object_name] = {version: obj} @@ -760,15 +787,15 @@ def get_bucket_versioning(self, bucket_name: str) -> VersioningConfig: bucket_name=bucket_name, object_name=None, ) - return self.buckets[bucket_name]["versioning"] + return self.buckets[bucket_name].versioning def list_objects( - self, - bucket_name, - prefix="", - recursive=False, - start_after="", - include_version=False, + self, + bucket_name, + prefix="", + recursive=False, + start_after="", + include_version=False, ): """ Lists objects in a bucket with the specified prefix and conditions. @@ -825,7 +852,9 @@ def _list_objects( bucket_name=bucket_name, object_name=obj_name, version_id=version, - is_delete_marker=bucket[obj_name][version].is_delete_marker, + is_delete_marker=bucket[obj_name][ + version + ].is_delete_marker, ) else: # only yield if the object is not a delete marker @@ -853,7 +882,12 @@ def _list_objects( object_name=None, ) return _list_objects( - self.buckets, bucket_name, prefix, recursive, start_after, include_version + self.buckets, + bucket_name, + prefix, + recursive, + start_after, + include_version, ) except Exception as e: raise e @@ -873,18 +907,20 @@ def remove_object(self, bucket_name, object_name, version_id=None): None: The method has no return value but indicates successful removal. """ self._health_check() - if object_name not in self.buckets[bucket_name]["objects"]: + if object_name not in self.buckets[bucket_name].objects: # object does not exist: nothing to do return try: if self.get_bucket_versioning(bucket_name).status == OFF: - del self.buckets[bucket_name]["objects"][object_name] + del self.buckets[bucket_name].objects[object_name] return if not version_id: if self.get_bucket_versioning(bucket_name).status == ENABLED: version_id = str(uuid4()) - self.buckets[bucket_name]["objects"][object_name][version_id] = MockMinioObject( + self.buckets[bucket_name].objects[object_name][ + version_id + ] = MockMinioObject( object_name=object_name, version_id=version_id, is_delete_marker=True, @@ -895,17 +931,19 @@ def remove_object(self, bucket_name, object_name, version_id=None): # That will lose the first version if it was created in an # unversioned bucket that was then versioned, # but that seems to be the expected behavior - obj = self.buckets[bucket_name]["objects"][object_name].pop(version_id) + obj = self.buckets[bucket_name].objects[object_name].pop(version_id) obj._is_delete_marker = True - self.buckets[bucket_name]["objects"][object_name][version_id] = obj + self.buckets[bucket_name].objects[object_name][version_id] = obj elif ( - version_id not in self.buckets[bucket_name]["objects"][object_name] - or self.buckets[bucket_name]["objects"][object_name][version_id].is_delete_marker + version_id not in self.buckets[bucket_name].objects[object_name] + or self.buckets[bucket_name] + .objects[object_name][version_id] + .is_delete_marker ): # version_id does not exist or is a delete_marker: nothing to do return else: - del self.buckets[bucket_name]["objects"][object_name][version_id] + del self.buckets[bucket_name].objects[object_name][version_id] except Exception: logging.error("remove_object(): Exception") logging.error(self.buckets) diff --git a/tests/test_minio_mock.py b/tests/test_minio_mock.py index 151235d..9e813b8 100644 --- a/tests/test_minio_mock.py +++ b/tests/test_minio_mock.py @@ -5,8 +5,38 @@ from minio import Minio from minio.commonconfig import ENABLED from minio.error import S3Error +from minio.versioningconfig import OFF from minio.versioningconfig import VersioningConfig +from pytest_minio_mock.plugin import MockMinioBucket + + +@pytest.mark.UNIT +class TestsMockMinioBucket: + @pytest.mark.UNIT + def test_init(self): + mock_minio_bucket = MockMinioBucket("a_bucket", None) + assert mock_minio_bucket._bucket_name == "a_bucket" + assert mock_minio_bucket._versioning == None + assert mock_minio_bucket._objects == {} + + versioning_config = VersioningConfig() + mock_minio_bucket = MockMinioBucket("a_bucket", versioning_config) + assert isinstance(mock_minio_bucket._versioning, VersioningConfig) + assert mock_minio_bucket.versioning.status == OFF + + @pytest.mark.UNIT + def test_versioning(self): + mock_minio_bucket = MockMinioBucket("a_bucket", VersioningConfig()) + versioning_config = mock_minio_bucket.versioning + assert isinstance(versioning_config, VersioningConfig) + assert versioning_config.status == OFF + versioning_config = VersioningConfig(status=ENABLED) + mock_minio_bucket.versioning = versioning_config + versioning_config = mock_minio_bucket.versioning + assert isinstance(versioning_config, VersioningConfig) + assert versioning_config.status == ENABLED + @pytest.mark.UNIT @pytest.mark.API From 8d7a756539200ac61ee1286c148c4c4601983c2a Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sat, 20 Apr 2024 14:45:21 +0200 Subject: [PATCH 04/26] updating the code to reflect adding the MockMinioBucket --- pytest_minio_mock/plugin.py | 48 ++++++++++++++++++------------------- tests/test_minio_mock.py | 7 +++--- 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/pytest_minio_mock/plugin.py b/pytest_minio_mock/plugin.py index 9c3b149..289ac82 100644 --- a/pytest_minio_mock/plugin.py +++ b/pytest_minio_mock/plugin.py @@ -98,15 +98,11 @@ def object_name(self): class MockMinioBucket: - def __init__(self, bucket_name: str, versioning: VersioningConfig): - self._bucket_name = bucket_name - self._objects = {} + def __init__(self, versioning: VersioningConfig, location=None, object_lock=False): self._versioning = versioning - - @property - def bucket_name(self): - """Get the name of the bucket.""" - return self._bucket_name + self._objects = {} + self._location = location + self._object_lock = object_lock @property def objects(self): @@ -571,21 +567,27 @@ def put_object( # objects created when versioning is suspended have a null version ID (None in Python) # I did not find the information about what exactly happend when versioning is OFF, # but I assume it is the same... - version = None - if self.get_bucket_versioning(bucket_name).status == ENABLED: - # If status is enabled, create a new version UUID - version = str(uuid4()) + # version = None + version = str(uuid4()) + # if self.get_bucket_versioning(bucket_name).status == ENABLED: + # If status is enabled, create a new version UUID + # version = str(uuid4()) - obj = MockMinioObject(object_name=object_name, data=data, version_id=version) + obj = MockMinioObject( + object_name=object_name, + data=data, + version_id=version, + is_delete_marker=False, + ) # If versioning is OFF, there can only be one version (None) of an object if self.get_bucket_versioning(bucket_name).status == OFF: - self.buckets[bucket_name]["objects"][object_name] = {version: obj} + self.buckets[bucket_name].objects[object_name] = {version: obj} else: - if object_name not in self.buckets[bucket_name]["objects"]: - self.buckets[bucket_name]["objects"][object_name] = {} + if object_name not in self.buckets[bucket_name].objects: + self.buckets[bucket_name].objects[object_name] = {} # If versioning is ENABLED, a new version is added. If it is SUSPENDED, # the None version is added (or replaced if already present) - self.buckets[bucket_name]["objects"][object_name][version] = obj + self.buckets[bucket_name].objects[object_name][version] = obj return "Upload successful" def get_presigned_url( @@ -742,13 +744,9 @@ def make_bucket(self, bucket_name, location=None, object_lock=False): bool: True indicating the bucket was successfully created. """ self._health_check() - self.buckets[bucket_name] = { - "versioning": VersioningConfig(), - "objects": {}, - # "__META__": - # {"name":bucket_name, - # "creation_date":datetime.datetime.utcnow()} - } + self.buckets[bucket_name] = MockMinioBucket( + versioning=VersioningConfig(), location=location, object_lock=object_lock + ) return True def set_bucket_versioning(self, bucket_name: str, config: VersioningConfig): @@ -769,7 +767,7 @@ def set_bucket_versioning(self, bucket_name: str, config: VersioningConfig): ) if not isinstance(config, VersioningConfig): raise ValueError("config must be VersioningConfig type") - self.buckets[bucket_name]["versioning"] = config + self.buckets[bucket_name].versioning = config def get_bucket_versioning(self, bucket_name: str) -> VersioningConfig: """Bucket versioning can be OFF (the initial value), ENABLED, or diff --git a/tests/test_minio_mock.py b/tests/test_minio_mock.py index 9e813b8..582f8be 100644 --- a/tests/test_minio_mock.py +++ b/tests/test_minio_mock.py @@ -15,19 +15,18 @@ class TestsMockMinioBucket: @pytest.mark.UNIT def test_init(self): - mock_minio_bucket = MockMinioBucket("a_bucket", None) - assert mock_minio_bucket._bucket_name == "a_bucket" + mock_minio_bucket = MockMinioBucket(None) assert mock_minio_bucket._versioning == None assert mock_minio_bucket._objects == {} versioning_config = VersioningConfig() - mock_minio_bucket = MockMinioBucket("a_bucket", versioning_config) + mock_minio_bucket = MockMinioBucket(versioning_config) assert isinstance(mock_minio_bucket._versioning, VersioningConfig) assert mock_minio_bucket.versioning.status == OFF @pytest.mark.UNIT def test_versioning(self): - mock_minio_bucket = MockMinioBucket("a_bucket", VersioningConfig()) + mock_minio_bucket = MockMinioBucket(VersioningConfig()) versioning_config = mock_minio_bucket.versioning assert isinstance(versioning_config, VersioningConfig) assert versioning_config.status == OFF From 11462b694a3324aefb1f9da77f23c5904ac4819d Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sat, 20 Apr 2024 14:46:08 +0200 Subject: [PATCH 05/26] added .pypirc to .gitignore as a safeguard --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index f9db1c3..669760d 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,6 @@ venv *.egg build dist + +#pypi +.pypirc From e23b519417b21a68916c448116cacc17a1d6a477 Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sat, 20 Apr 2024 14:49:45 +0200 Subject: [PATCH 06/26] pytest: 14 passing, 2 failing --- pytest_minio_mock/plugin.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pytest_minio_mock/plugin.py b/pytest_minio_mock/plugin.py index 289ac82..032e0ce 100644 --- a/pytest_minio_mock/plugin.py +++ b/pytest_minio_mock/plugin.py @@ -821,11 +821,11 @@ def _list_objects( include_version=False, ): # Initialization - bucket = buckets[bucket_name]["objects"] + bucket_objects = buckets[bucket_name].objects # bucket_objects = [] seen_prefixes = set() - for obj_name in bucket.keys(): + for obj_name in bucket_objects.keys(): if obj_name.startswith(prefix) and ( start_after == "" or obj_name > start_after ): @@ -845,18 +845,20 @@ def _list_objects( # Directly add the object for recursive listing or if it's a file in the current directory if include_version: # Minio API always includes deleted markers if include_version - for version in bucket[obj_name]: + for version in bucket_objects[obj_name]: yield Object( bucket_name=bucket_name, object_name=obj_name, version_id=version, - is_delete_marker=bucket[obj_name][ + is_delete_marker=bucket_objects[obj_name][ version ].is_delete_marker, ) else: # only yield if the object is not a delete marker - obj = bucket[obj_name][list(bucket[obj_name].keys())[-1]] + obj = bucket_objects[obj_name][ + list(bucket_objects[obj_name].keys())[-1] + ] if not obj.is_delete_marker: yield Object( bucket_name=bucket_name, From 905b1595c429bbb3492cc92796b12d724493dcc9 Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sat, 20 Apr 2024 15:44:31 +0200 Subject: [PATCH 07/26] progress --- pytest_minio_mock/plugin.py | 125 ++++++++++++++++++++++-------------- tests/test_minio_mock.py | 7 +- 2 files changed, 80 insertions(+), 52 deletions(-) diff --git a/pytest_minio_mock/plugin.py b/pytest_minio_mock/plugin.py index 032e0ce..4002b18 100644 --- a/pytest_minio_mock/plugin.py +++ b/pytest_minio_mock/plugin.py @@ -86,6 +86,10 @@ def is_delete_marker(self): """Is the object deleted n a versioned bucket ?""" return self._is_delete_marker + @is_delete_marker.setter + def is_delete_marker(self, value): + self._is_delete_marker = value + @data.setter def data(self, value): """Set the data the object contains.""" @@ -366,20 +370,19 @@ def get_object( HTTPResponse: A response object containing the object data. """ self._health_check() - nosuchkey = S3Error( - message="The specified key does not exist.", - resource=f"/{bucket_name}/{object_name}", - request_id=None, - host_id=None, - response="mocked_response", - code=404, - bucket_name=bucket_name, - object_name=object_name, - ) try: the_object = self.buckets[bucket_name].objects[object_name] except KeyError: - raise nosuchkey + raise S3Error( + message="The specified key does not exist.", + resource=f"/{bucket_name}/{object_name}", + request_id=None, + host_id=None, + response="mocked_response", + code=404, + bucket_name=bucket_name, + object_name=object_name, + ) if version_id and not validators.uuid(version_id): raise S3Error( @@ -409,7 +412,17 @@ def get_object( found_obj = obj break if not found_obj: - raise nosuchkey + raise S3Error( + message="The specified key does not exist.", + resource=f"/{bucket_name}/{object_name}", + request_id=None, + host_id=None, + response="mocked_response", + code=404, + bucket_name=bucket_name, + object_name=object_name, + ) + try: the_object = the_object[version_id] except KeyError: @@ -436,6 +449,7 @@ def get_object( object_name=object_name, ) data = the_object.data + # Create a buffer containing the data if isinstance(data, io.BytesIO): body = copy.deepcopy(data) @@ -567,27 +581,27 @@ def put_object( # objects created when versioning is suspended have a null version ID (None in Python) # I did not find the information about what exactly happend when versioning is OFF, # but I assume it is the same... + # version = None - version = str(uuid4()) # if self.get_bucket_versioning(bucket_name).status == ENABLED: - # If status is enabled, create a new version UUID - # version = str(uuid4()) + # # If status is enabled, create a new version UUID + # version = str(uuid4()) + version = str(uuid4()) obj = MockMinioObject( object_name=object_name, data=data, version_id=version, is_delete_marker=False, ) - # If versioning is OFF, there can only be one version (None) of an object + # If versioning is OFF, there can only be one version of an object (store a read version_id non-the-less) if self.get_bucket_versioning(bucket_name).status == OFF: self.buckets[bucket_name].objects[object_name] = {version: obj} else: if object_name not in self.buckets[bucket_name].objects: self.buckets[bucket_name].objects[object_name] = {} - # If versioning is ENABLED, a new version is added. If it is SUSPENDED, - # the None version is added (or replaced if already present) self.buckets[bucket_name].objects[object_name][version] = obj + return "Upload successful" def get_presigned_url( @@ -863,7 +877,7 @@ def _list_objects( yield Object( bucket_name=bucket_name, object_name=obj_name, - version_id=obj.version_id, + version_id=None, is_delete_marker=False, ) @@ -912,38 +926,51 @@ def remove_object(self, bucket_name, object_name, version_id=None): return try: if self.get_bucket_versioning(bucket_name).status == OFF: - del self.buckets[bucket_name].objects[object_name] - return + if version_id: + if version_id in self.buckets[bucket_name].objects[object_name]: + del self.buckets[bucket_name].objects[object_name][version_id] + else: + # nothing to do + return + else: + del self.buckets[bucket_name].objects[object_name] + return + except Exception: + logging.error("remove_object(): Exception") + logging.error(self.buckets) + raise - if not version_id: - if self.get_bucket_versioning(bucket_name).status == ENABLED: - version_id = str(uuid4()) - self.buckets[bucket_name].objects[object_name][ - version_id - ] = MockMinioObject( - object_name=object_name, - version_id=version_id, - is_delete_marker=True, - data=b"", - ) + try: + if self.get_bucket_versioning(bucket_name).status == ENABLED: + if version_id: + if version_id not in self.buckets[bucket_name].objects[object_name]: + # version_id does not exist + # nothing to do + return + elif ( + self.buckets[bucket_name] + .objects[object_name][version_id] + .is_delete_marker + == True + ): + # version_id exists but delete_market is already True + # nothing to do + return + else: + # mark the object as deleted + self.buckets[bucket_name].objects[object_name][ + version_id + ].is_delete_marker = True else: - # Ensures the None version is put as last version. - # That will lose the first version if it was created in an - # unversioned bucket that was then versioned, - # but that seems to be the expected behavior - obj = self.buckets[bucket_name].objects[object_name].pop(version_id) - obj._is_delete_marker = True - self.buckets[bucket_name].objects[object_name][version_id] = obj - elif ( - version_id not in self.buckets[bucket_name].objects[object_name] - or self.buckets[bucket_name] - .objects[object_name][version_id] - .is_delete_marker - ): - # version_id does not exist or is a delete_marker: nothing to do - return - else: - del self.buckets[bucket_name].objects[object_name][version_id] + # remove the lastest object by setting the delete marker to True + # get the latest object version_id + latest_object_version_id = list( + self.buckets[bucket_name].objects[object_name].keys() + )[-1] + self.buckets[bucket_name].objects[object_name][ + latest_object_version_id + ].is_delete_marker = True + except Exception: logging.error("remove_object(): Exception") logging.error(self.buckets) diff --git a/tests/test_minio_mock.py b/tests/test_minio_mock.py index 582f8be..255a794 100644 --- a/tests/test_minio_mock.py +++ b/tests/test_minio_mock.py @@ -6,6 +6,7 @@ from minio.commonconfig import ENABLED from minio.error import S3Error from minio.versioningconfig import OFF +from minio.versioningconfig import SUSPENDED from minio.versioningconfig import VersioningConfig from pytest_minio_mock.plugin import MockMinioBucket @@ -58,11 +59,11 @@ def test_adding_and_removing_objects(minio_mock): client.fput_object(bucket_name, object_name, file_path) assert ( - object_name in client.buckets[bucket_name]["objects"] + object_name in client.buckets[bucket_name].objects ), "Object should be in the bucket after upload" client.remove_object(bucket_name, object_name) - assert object_name not in client.buckets[bucket_name]["objects"] + assert object_name not in client.buckets[bucket_name].objects @pytest.mark.UNIT @@ -130,7 +131,7 @@ def test_versioned_objects_after_upload(minio_mock): assert len(objects) == 2 assert objects[0].version_id is None assert last_version is not None - client.set_bucket_versioning(bucket_name, VersioningConfig("Suspended")) + client.set_bucket_versioning(bucket_name, VersioningConfig(SUSPENDED)) client.remove_object(bucket_name, object_name) objects = list(client.list_objects(bucket_name, object_name, include_version=True)) From a6ff734cf3fdedd46c9f1f275657a374e3ceda43 Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sat, 20 Apr 2024 16:31:30 +0200 Subject: [PATCH 08/26] using null instead of None --- pytest_minio_mock/plugin.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/pytest_minio_mock/plugin.py b/pytest_minio_mock/plugin.py index 4002b18..e52b987 100644 --- a/pytest_minio_mock/plugin.py +++ b/pytest_minio_mock/plugin.py @@ -302,7 +302,7 @@ def fget_object( file_path, request_headers=None, sse=None, - version_id=None, + version_id="null", extra_query_params=None, ): """ @@ -345,7 +345,7 @@ def get_object( length: int = 0, request_headers=None, ssec=None, - version_id=None, + version_id="null", extra_query_params=None, ): """ @@ -384,7 +384,7 @@ def get_object( object_name=object_name, ) - if version_id and not validators.uuid(version_id): + if version_id and version_id != "null" and not validators.uuid(version_id): raise S3Error( message="Invalid version id specified", resource=f"/{bucket_name}/{object_name}", @@ -404,7 +404,7 @@ def get_object( # If no version was specified, try to find the first one that does not # correspond to a deleted object. Note that it can still be None after # that if versioning is Off, but that is not a problem. - if not version_id: + if version_id == "null": found_obj = None # reversed to start by the newest object for version_id, obj in reversed(the_object.items()): @@ -582,12 +582,11 @@ def put_object( # I did not find the information about what exactly happend when versioning is OFF, # but I assume it is the same... - # version = None - # if self.get_bucket_versioning(bucket_name).status == ENABLED: - # # If status is enabled, create a new version UUID - # version = str(uuid4()) + version = "null" + # If status is enabled, create a new version UUID + if self.get_bucket_versioning(bucket_name).status == ENABLED: + version = str(uuid4()) - version = str(uuid4()) obj = MockMinioObject( object_name=object_name, data=data, @@ -877,7 +876,7 @@ def _list_objects( yield Object( bucket_name=bucket_name, object_name=obj_name, - version_id=None, + version_id="null", is_delete_marker=False, ) @@ -906,7 +905,7 @@ def _list_objects( except Exception as e: raise e - def remove_object(self, bucket_name, object_name, version_id=None): + def remove_object(self, bucket_name, object_name, version_id="null"): """ Removes an object from a bucket in the mock Minio server. From 5c66aa084dda69c199a8717e2be552664a6fd90f Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sat, 20 Apr 2024 16:39:35 +0200 Subject: [PATCH 09/26] added ._latest --- pytest_minio_mock/plugin.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pytest_minio_mock/plugin.py b/pytest_minio_mock/plugin.py index e52b987..311b0e8 100644 --- a/pytest_minio_mock/plugin.py +++ b/pytest_minio_mock/plugin.py @@ -55,7 +55,7 @@ class MockMinioObject: versioned bucket """ - def __init__(self, object_name, data, version_id, is_delete_marker): + def __init__(self, object_name, data, version_id, is_delete_marker, latest): """ Initialize the MockMinioObject with a name and data. @@ -70,6 +70,8 @@ def __init__(self, object_name, data, version_id, is_delete_marker): self._data = data self._version_id = version_id self._is_delete_marker = is_delete_marker + self._last_modified = datetime.datetime.now() + self._latest = latest @property def data(self): @@ -592,6 +594,7 @@ def put_object( data=data, version_id=version, is_delete_marker=False, + latest=True, ) # If versioning is OFF, there can only be one version of an object (store a read version_id non-the-less) if self.get_bucket_versioning(bucket_name).status == OFF: @@ -599,6 +602,9 @@ def put_object( else: if object_name not in self.buckets[bucket_name].objects: self.buckets[bucket_name].objects[object_name] = {} + else: + for version, obj in self.buckets[bucket_name].objects[object_name]: + obj._latest = False self.buckets[bucket_name].objects[object_name][version] = obj return "Upload successful" From acb17e664afaad3644a1d0743b44d12f2df378d5 Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sat, 20 Apr 2024 17:19:33 +0200 Subject: [PATCH 10/26] working on fixing list_objects --- pytest_minio_mock/plugin.py | 48 ++++++++++++++++++++++++------------- tests/test_minio_mock.py | 8 ++++--- 2 files changed, 36 insertions(+), 20 deletions(-) diff --git a/pytest_minio_mock/plugin.py b/pytest_minio_mock/plugin.py index 311b0e8..0bbbe30 100644 --- a/pytest_minio_mock/plugin.py +++ b/pytest_minio_mock/plugin.py @@ -55,7 +55,7 @@ class MockMinioObject: versioned bucket """ - def __init__(self, object_name, data, version_id, is_delete_marker, latest): + def __init__(self, object_name, data, version_id, is_delete_marker, is_latest): """ Initialize the MockMinioObject with a name and data. @@ -70,8 +70,8 @@ def __init__(self, object_name, data, version_id, is_delete_marker, latest): self._data = data self._version_id = version_id self._is_delete_marker = is_delete_marker - self._last_modified = datetime.datetime.now() - self._latest = latest + self._last_modified = datetime.datetime.utcnow() + self._is_latest = is_latest @property def data(self): @@ -83,6 +83,11 @@ def version_id(self): """Get the version of the object.""" return self._version_id + @property + def last_modified(self): + """return the last modified datetime of the object.""" + return self._last_modified + @property def is_delete_marker(self): """Is the object deleted n a versioned bucket ?""" @@ -102,6 +107,11 @@ def object_name(self): """Get the name of the object.""" return self._object_name + @property + def is_latest(self): + """is this object marked latest""" + return self._is_latest + class MockMinioBucket: def __init__(self, versioning: VersioningConfig, location=None, object_lock=False): @@ -594,7 +604,7 @@ def put_object( data=data, version_id=version, is_delete_marker=False, - latest=True, + is_latest=True, ) # If versioning is OFF, there can only be one version of an object (store a read version_id non-the-less) if self.get_bucket_versioning(bucket_name).status == OFF: @@ -603,8 +613,8 @@ def put_object( if object_name not in self.buckets[bucket_name].objects: self.buckets[bucket_name].objects[object_name] = {} else: - for version, obj in self.buckets[bucket_name].objects[object_name]: - obj._latest = False + for obj in self.buckets[bucket_name].objects[object_name].values(): + obj._is_latest = False self.buckets[bucket_name].objects[object_name][version] = obj return "Upload successful" @@ -864,30 +874,34 @@ def _list_objects( # Directly add the object for recursive listing or if it's a file in the current directory if include_version: # Minio API always includes deleted markers if include_version - for version in bucket_objects[obj_name]: + for version, obj in reversed( + sorted( + bucket_objects[obj_name].items(), + key=lambda i: i[1].last_modified, + ) + ): yield Object( bucket_name=bucket_name, object_name=obj_name, + last_modified=obj.last_modified, version_id=version, - is_delete_marker=bucket_objects[obj_name][ - version - ].is_delete_marker, + is_latest="true" if obj.is_latest else "false", + is_delete_marker=obj.is_delete_marker, ) else: # only yield if the object is not a delete marker - obj = bucket_objects[obj_name][ - list(bucket_objects[obj_name].keys())[-1] - ] + version = "null" + obj = bucket_objects[obj_name][version] if not obj.is_delete_marker: yield Object( bucket_name=bucket_name, object_name=obj_name, - version_id="null", - is_delete_marker=False, + last_modified=obj.last_modified, + version_id=None, + is_latest=None, + is_delete_marker=obj.is_delete_marker, ) - # return bucket_objects - try: if bucket_name not in self.buckets: raise S3Error( diff --git a/tests/test_minio_mock.py b/tests/test_minio_mock.py index 255a794..4b91214 100644 --- a/tests/test_minio_mock.py +++ b/tests/test_minio_mock.py @@ -78,6 +78,7 @@ def test_versioned_objects(minio_mock): client.set_bucket_versioning(bucket_name, VersioningConfig(ENABLED)) client.fput_object(bucket_name, object_name, file_path) client.fput_object(bucket_name, object_name, file_path) + # list_objects should sort by newest objects = list(client.list_objects(bucket_name, object_name, include_version=True)) assert len(objects) == 2 first_version = objects[0].version_id @@ -123,13 +124,14 @@ def test_versioned_objects_after_upload(minio_mock): objects = list(client.list_objects(bucket_name, object_name, include_version=True)) assert len(objects) == 1 first_version = objects[0].version_id - assert first_version is None + assert first_version is "null" + client.fput_object(bucket_name, object_name, file_path) client.fput_object(bucket_name, object_name, file_path) objects = list(client.list_objects(bucket_name, object_name, include_version=True)) last_version = objects[1].version_id - assert len(objects) == 2 - assert objects[0].version_id is None + assert len(objects) == 3 + assert objects[-1].version_id is "null" assert last_version is not None client.set_bucket_versioning(bucket_name, VersioningConfig(SUSPENDED)) From 1a66f0eff5a081949ff6a843ac18c0006eff42ba Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sat, 20 Apr 2024 17:55:38 +0200 Subject: [PATCH 11/26] fixing list_objects... --- pytest_minio_mock/plugin.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/pytest_minio_mock/plugin.py b/pytest_minio_mock/plugin.py index 0bbbe30..ddcaad7 100644 --- a/pytest_minio_mock/plugin.py +++ b/pytest_minio_mock/plugin.py @@ -613,8 +613,10 @@ def put_object( if object_name not in self.buckets[bucket_name].objects: self.buckets[bucket_name].objects[object_name] = {} else: - for obj in self.buckets[bucket_name].objects[object_name].values(): - obj._is_latest = False + for old_version in self.buckets[bucket_name].objects[object_name]: + self.buckets[bucket_name].objects[object_name][ + old_version + ]._is_latest = False self.buckets[bucket_name].objects[object_name][version] = obj return "Upload successful" @@ -874,23 +876,31 @@ def _list_objects( # Directly add the object for recursive listing or if it's a file in the current directory if include_version: # Minio API always includes deleted markers if include_version - for version, obj in reversed( - sorted( - bucket_objects[obj_name].items(), - key=lambda i: i[1].last_modified, + versions_list = list( + reversed( + sorted( + bucket_objects[obj_name].items(), + key=lambda i: i[1].last_modified, + ) ) - ): + ) + for version, obj in versions_list: yield Object( bucket_name=bucket_name, object_name=obj_name, last_modified=obj.last_modified, version_id=version, - is_latest="true" if obj.is_latest else "false", + is_latest=obj.is_latest, is_delete_marker=obj.is_delete_marker, ) else: # only yield if the object is not a delete marker - version = "null" + version, _ = list( + sorted( + bucket_objects[obj_name].items(), + key=lambda i: i[1].last_modified, + ) + )[-1] obj = bucket_objects[obj_name][version] if not obj.is_delete_marker: yield Object( From ac9c3f6478e7b8102b8a06bc4b45dea216a6ff51 Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sat, 20 Apr 2024 23:42:50 +0200 Subject: [PATCH 12/26] fixing removing objects and listing objects --- pytest_minio_mock/plugin.py | 2 +- tests/test_minio_mock.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pytest_minio_mock/plugin.py b/pytest_minio_mock/plugin.py index ddcaad7..05cee20 100644 --- a/pytest_minio_mock/plugin.py +++ b/pytest_minio_mock/plugin.py @@ -955,7 +955,7 @@ def remove_object(self, bucket_name, object_name, version_id="null"): return try: if self.get_bucket_versioning(bucket_name).status == OFF: - if version_id: + if version_id and version_id != "null": if version_id in self.buckets[bucket_name].objects[object_name]: del self.buckets[bucket_name].objects[object_name][version_id] else: diff --git a/tests/test_minio_mock.py b/tests/test_minio_mock.py index 4b91214..8903638 100644 --- a/tests/test_minio_mock.py +++ b/tests/test_minio_mock.py @@ -49,7 +49,8 @@ def test_make_bucket(minio_mock): @pytest.mark.UNIT @pytest.mark.API -def test_adding_and_removing_objects(minio_mock): +def test_adding_and_removing_objects_basic(minio_mock): + # simple thing bucket_name = "test-bucket" object_name = "test-object" file_path = "tests/fixtures/maya.jpeg" From b75d16ccd53e9270f3748262b33b96b23f7fe454 Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sun, 21 Apr 2024 03:11:03 +0200 Subject: [PATCH 13/26] progress fixing list_objects and remove_object --- pytest_minio_mock/plugin.py | 180 ++++++++++++++++++++++++------------ tests/test_minio_mock.py | 33 ++++++- 2 files changed, 151 insertions(+), 62 deletions(-) diff --git a/pytest_minio_mock/plugin.py b/pytest_minio_mock/plugin.py index 05cee20..f057daa 100644 --- a/pytest_minio_mock/plugin.py +++ b/pytest_minio_mock/plugin.py @@ -120,6 +120,14 @@ def __init__(self, versioning: VersioningConfig, location=None, object_lock=Fals self._location = location self._object_lock = object_lock + def _get_latest_object(self, object_name): + """returns the latest_object""" + for version in self._objects[object_name]: + obj = self._objects[object_name][version] + if obj.is_latest: + return obj + raise Exception("no latest") + @property def objects(self): """Get the objects stored in the bucket.""" @@ -413,16 +421,27 @@ def get_object( # interact with them. And if versioning is Off, then None is a valid # version too, and we can work with it - # If no version was specified, try to find the first one that does not + # If version == "null", try to find the first one that does not # correspond to a deleted object. Note that it can still be None after # that if versioning is Off, but that is not a problem. if version_id == "null": found_obj = None # reversed to start by the newest object + # + # + # + # + # for version_id, obj in reversed(the_object.items()): if not obj.is_delete_marker: found_obj = obj break + # + # + # + # + # + # if not found_obj: raise S3Error( message="The specified key does not exist.", @@ -538,7 +557,7 @@ def fput_object( part_size=part_size, ) - def put_object( + def _put_object( self, bucket_name: str, object_name: str, @@ -548,33 +567,13 @@ def put_object( metadata=None, sse=None, progress=None, - part_size: int = 0 + part_size: int = 0, # num_parallel_uploads: int = 3, # tags = None, # retention = None, - # legal_hold: bool = False + # legal_hold: bool = False, + is_delete_marker=False, ): - """ - Simulates uploading an object to the mock Minio server. - - Stores an object in a specified bucket with the given data. - - Args: - bucket_name (str): The name of the bucket to upload to. - object_name (str): The object name to create in the bucket. - data: The data to upload. Can be bytes or a file-like object. - length (int): The length of the data to upload. - content_type (str, optional): The content type of the object. Defaults to - "application/octet-stream". - metadata (dict, optional): A dictionary of additional metadata for the object. Defaults - to None. - sse (optional): Server-side encryption option. Defaults to None. - progress (optional): Callback function to monitor progress. Defaults to None. - part_size (int, optional): The size of each part in multi-part upload. Defaults to 0. - - Returns: - str: Confirmation message indicating successful upload. - """ self._health_check() if not self.bucket_exists(bucket_name): raise S3Error( @@ -590,9 +589,7 @@ def put_object( # According to # https://min.io/docs/minio/linux/administration/object-management/object-versioning.html#suspend-bucket-versioning - # objects created when versioning is suspended have a null version ID (None in Python) - # I did not find the information about what exactly happend when versioning is OFF, - # but I assume it is the same... + # objects created when versioning is suspended have a 'null' version ID (None in Python) version = "null" # If status is enabled, create a new version UUID @@ -603,7 +600,7 @@ def put_object( object_name=object_name, data=data, version_id=version, - is_delete_marker=False, + is_delete_marker=is_delete_marker, is_latest=True, ) # If versioning is OFF, there can only be one version of an object (store a read version_id non-the-less) @@ -618,6 +615,56 @@ def put_object( old_version ]._is_latest = False self.buckets[bucket_name].objects[object_name][version] = obj + return obj + + def put_object( + self, + bucket_name: str, + object_name: str, + data, + length: int, + content_type: str = "application/octet-stream", + metadata=None, + sse=None, + progress=None, + part_size: int = 0 + # num_parallel_uploads: int = 3, + # tags = None, + # retention = None, + # legal_hold: bool = False, + ): + """ + Simulates uploading an object to the mock Minio server. + + Stores an object in a specified bucket with the given data. + + Args: + bucket_name (str): The name of the bucket to upload to. + object_name (str): The object name to create in the bucket. + data: The data to upload. Can be bytes or a file-like object. + length (int): The length of the data to upload. + content_type (str, optional): The content type of the object. Defaults to + "application/octet-stream". + metadata (dict, optional): A dictionary of additional metadata for the object. Defaults + to None. + sse (optional): Server-side encryption option. Defaults to None. + progress (optional): Callback function to monitor progress. Defaults to None. + part_size (int, optional): The size of each part in multi-part upload. Defaults to 0. + + Returns: + str: Confirmation message indicating successful upload. + """ + obj = self._put_object( + bucket_name, + object_name, + data, + length, + content_type, + metadata, + sse, + progress, + part_size, + ) return "Upload successful" @@ -875,13 +922,15 @@ def _list_objects( continue # Skip further processing to prevent adding the full object path # Directly add the object for recursive listing or if it's a file in the current directory if include_version: - # Minio API always includes deleted markers if include_version + # Minio API always sort versions by time, it also includes delete markers at the end newwst first + ################################################################################################## versions_list = list( - reversed( - sorted( - bucket_objects[obj_name].items(), - key=lambda i: i[1].last_modified, - ) + sorted( + bucket_objects[obj_name].items(), + key=lambda i: ( + i[1].is_delete_marker, + -i[1].last_modified.timestamp(), + ), ) ) for version, obj in versions_list: @@ -890,7 +939,7 @@ def _list_objects( object_name=obj_name, last_modified=obj.last_modified, version_id=version, - is_latest=obj.is_latest, + is_latest="true" if obj.is_latest else "false", is_delete_marker=obj.is_delete_marker, ) else: @@ -935,7 +984,7 @@ def _list_objects( except Exception as e: raise e - def remove_object(self, bucket_name, object_name, version_id="null"): + def remove_object(self, bucket_name, object_name, version_id=None): """ Removes an object from a bucket in the mock Minio server. @@ -955,7 +1004,7 @@ def remove_object(self, bucket_name, object_name, version_id="null"): return try: if self.get_bucket_versioning(bucket_name).status == OFF: - if version_id and version_id != "null": + if version_id: if version_id in self.buckets[bucket_name].objects[object_name]: del self.buckets[bucket_name].objects[object_name][version_id] else: @@ -976,33 +1025,44 @@ def remove_object(self, bucket_name, object_name, version_id="null"): # version_id does not exist # nothing to do return - elif ( - self.buckets[bucket_name] - .objects[object_name][version_id] - .is_delete_marker - == True - ): - # version_id exists but delete_market is already True - # nothing to do - return + # elif ( + # self.buckets[bucket_name] + # .objects[object_name][version_id] + # .is_delete_marker + # == True + # ): + # de + # version_id exists but delete_market is already True + # nothing to do + # raise Exception("TODO") else: # mark the object as deleted - self.buckets[bucket_name].objects[object_name][ - version_id - ].is_delete_marker = True - else: - # remove the lastest object by setting the delete marker to True - # get the latest object version_id - latest_object_version_id = list( - self.buckets[bucket_name].objects[object_name].keys() - )[-1] - self.buckets[bucket_name].objects[object_name][ - latest_object_version_id - ].is_delete_marker = True + # self.buckets[bucket_name].objects[object_name][ + # version_id + # ].is_delete_marker = True + del self.buckets[bucket_name].objects[object_name][version_id] - except Exception: + else: # version_id is False + # + latest_obj = self.buckets[bucket_name]._get_latest_object( + object_name + ) + if latest_obj.is_delete_marker: + # nothing to do + return + + self._put_object( + bucket_name=bucket_name, + object_name=object_name, + data=b"", + length=0, + is_delete_marker=True, + ) + + except Exception as e: logging.error("remove_object(): Exception") logging.error(self.buckets) + logging.error(e) raise diff --git a/tests/test_minio_mock.py b/tests/test_minio_mock.py index 8903638..121f8cb 100644 --- a/tests/test_minio_mock.py +++ b/tests/test_minio_mock.py @@ -63,7 +63,6 @@ def test_adding_and_removing_objects_basic(minio_mock): object_name in client.buckets[bucket_name].objects ), "Object should be in the bucket after upload" client.remove_object(bucket_name, object_name) - assert object_name not in client.buckets[bucket_name].objects @@ -76,20 +75,50 @@ def test_versioned_objects(minio_mock): client = Minio("http://local.host:9000") client.make_bucket(bucket_name) + + client.fput_object(bucket_name, object_name, file_path) + objects = list(client.list_objects(bucket_name, object_name, include_version=True)) + assert len(objects) == 1 + client.remove_object(bucket_name, object_name, version_id="null") + objects = list(client.list_objects(bucket_name, object_name, include_version=True)) + assert len(objects) == 0 + + # Versioning Enabled client.set_bucket_versioning(bucket_name, VersioningConfig(ENABLED)) + # Add two objects client.fput_object(bucket_name, object_name, file_path) client.fput_object(bucket_name, object_name, file_path) # list_objects should sort by newest objects = list(client.list_objects(bucket_name, object_name, include_version=True)) assert len(objects) == 2 + first_version = objects[0].version_id last_version = objects[1].version_id client.remove_object(bucket_name, object_name, version_id=first_version) objects = list(client.list_objects(bucket_name, object_name, include_version=True)) assert len(objects) == 1 + assert objects[0].version_id == last_version + + client.fput_object(bucket_name, object_name, file_path) + objects = list(client.list_objects(bucket_name, object_name, include_version=True)) + assert len(objects) == 2 first_version = objects[0].version_id - assert first_version == last_version + assert first_version != last_version + + client.remove_object(bucket_name, object_name) + objects = list(client.list_objects(bucket_name, object_name, include_version=True)) + assert len(objects) == 3 + client.remove_object(bucket_name, object_name) + objects = list(client.list_objects(bucket_name, object_name, include_version=True)) + assert len(objects) == 3 + + client.fput_object(bucket_name, object_name, file_path) + objects = list(client.list_objects(bucket_name, object_name, include_version=True)) + assert len(objects) == 4 + client.remove_object(bucket_name, object_name) + objects = list(client.list_objects(bucket_name, object_name, include_version=True)) + assert len(objects) == 5 client.fput_object(bucket_name, object_name, file_path) objects = list(client.list_objects(bucket_name, object_name, include_version=True)) From 16782c3647125c6761388b02d7bbbc63e922d4b6 Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sun, 21 Apr 2024 03:36:16 +0200 Subject: [PATCH 14/26] fixing list_objects when include_versions=False --- pytest_minio_mock/plugin.py | 84 ++++++++++++------------------------- tests/test_minio_mock.py | 10 +++++ 2 files changed, 37 insertions(+), 57 deletions(-) diff --git a/pytest_minio_mock/plugin.py b/pytest_minio_mock/plugin.py index f057daa..74b5f5a 100644 --- a/pytest_minio_mock/plugin.py +++ b/pytest_minio_mock/plugin.py @@ -120,7 +120,7 @@ def __init__(self, versioning: VersioningConfig, location=None, object_lock=Fals self._location = location self._object_lock = object_lock - def _get_latest_object(self, object_name): + def get_latest_object(self, object_name): """returns the latest_object""" for version in self._objects[object_name]: obj = self._objects[object_name][version] @@ -322,7 +322,7 @@ def fget_object( file_path, request_headers=None, sse=None, - version_id="null", + version_id: str | None = None, extra_query_params=None, ): """ @@ -365,7 +365,7 @@ def get_object( length: int = 0, request_headers=None, ssec=None, - version_id="null", + version_id: str | None = None, extra_query_params=None, ): """ @@ -421,38 +421,15 @@ def get_object( # interact with them. And if versioning is Off, then None is a valid # version too, and we can work with it - # If version == "null", try to find the first one that does not + # If version == None, try to find the first one that does not # correspond to a deleted object. Note that it can still be None after # that if versioning is Off, but that is not a problem. - if version_id == "null": - found_obj = None - # reversed to start by the newest object - # - # - # - # - # - for version_id, obj in reversed(the_object.items()): - if not obj.is_delete_marker: - found_obj = obj - break - # - # - # - # - # - # - if not found_obj: - raise S3Error( - message="The specified key does not exist.", - resource=f"/{bucket_name}/{object_name}", - request_id=None, - host_id=None, - response="mocked_response", - code=404, - bucket_name=bucket_name, - object_name=object_name, - ) + if not version_id: + if self.get_bucket_versioning(bucket_name).status == OFF: + version_id = "null" + else: + latest_object = self.buckets[bucket_name].get_latest_object(object_name) + version_id = latest_object.version_id try: the_object = the_object[version_id] @@ -467,19 +444,19 @@ def get_object( bucket_name=bucket_name, object_name=object_name, ) - finally: - if the_object.is_delete_marker: - raise S3Error( - message="The specified method is not allowed against this resource.", - resource=f"/{bucket_name}/{object_name}", - request_id=None, - host_id=None, - response="mocked_response", - code=403, - bucket_name=bucket_name, - object_name=object_name, - ) - data = the_object.data + + if the_object.is_delete_marker: + raise S3Error( + message="The specified method is not allowed against this resource.", + resource=f"/{bucket_name}/{object_name}", + request_id=None, + host_id=None, + response="mocked_response", + code=403, + bucket_name=bucket_name, + object_name=object_name, + ) + data = the_object.data # Create a buffer containing the data if isinstance(data, io.BytesIO): @@ -676,7 +653,7 @@ def get_presigned_url( expires=datetime.timedelta(days=7), response_headers=None, request_date=None, - version_id=None, + version_id: str | None = None, extra_query_params=None, ): """ @@ -732,7 +709,7 @@ def presigned_get_object( expires=datetime.timedelta(days=7), response_headers=None, request_date=None, - version_id=None, + version_id: str | None = None, extra_query_params=None, ): """ @@ -923,7 +900,6 @@ def _list_objects( # Directly add the object for recursive listing or if it's a file in the current directory if include_version: # Minio API always sort versions by time, it also includes delete markers at the end newwst first - ################################################################################################## versions_list = list( sorted( bucket_objects[obj_name].items(), @@ -944,13 +920,7 @@ def _list_objects( ) else: # only yield if the object is not a delete marker - version, _ = list( - sorted( - bucket_objects[obj_name].items(), - key=lambda i: i[1].last_modified, - ) - )[-1] - obj = bucket_objects[obj_name][version] + obj = buckets[bucket_name].get_latest_object(obj_name) if not obj.is_delete_marker: yield Object( bucket_name=bucket_name, @@ -1044,7 +1014,7 @@ def remove_object(self, bucket_name, object_name, version_id=None): else: # version_id is False # - latest_obj = self.buckets[bucket_name]._get_latest_object( + latest_obj = self.buckets[bucket_name].get_latest_object( object_name ) if latest_obj.is_delete_marker: diff --git a/tests/test_minio_mock.py b/tests/test_minio_mock.py index 121f8cb..b7fe1df 100644 --- a/tests/test_minio_mock.py +++ b/tests/test_minio_mock.py @@ -113,13 +113,23 @@ def test_versioned_objects(minio_mock): objects = list(client.list_objects(bucket_name, object_name, include_version=True)) assert len(objects) == 3 + objects = list(client.list_objects(bucket_name, object_name)) + assert len(objects) == 0 + client.fput_object(bucket_name, object_name, file_path) objects = list(client.list_objects(bucket_name, object_name, include_version=True)) assert len(objects) == 4 + + objects = list(client.list_objects(bucket_name, object_name)) + assert len(objects) == 1 + client.remove_object(bucket_name, object_name) objects = list(client.list_objects(bucket_name, object_name, include_version=True)) assert len(objects) == 5 + objects = list(client.list_objects(bucket_name, object_name)) + assert len(objects) == 0 + client.fput_object(bucket_name, object_name, file_path) objects = list(client.list_objects(bucket_name, object_name, include_version=True)) assert len(objects) == 2 From 35b192b1b076ae13d250aada32f11ed0fdf2dba7 Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sun, 21 Apr 2024 04:33:34 +0200 Subject: [PATCH 15/26] progress in remove_objects with versioning --- pytest_minio_mock/plugin.py | 67 +++++++++++++++++++++++++------------ tests/test_minio_mock.py | 30 ++++++----------- 2 files changed, 55 insertions(+), 42 deletions(-) diff --git a/pytest_minio_mock/plugin.py b/pytest_minio_mock/plugin.py index 74b5f5a..e2642d0 100644 --- a/pytest_minio_mock/plugin.py +++ b/pytest_minio_mock/plugin.py @@ -122,6 +122,8 @@ def __init__(self, versioning: VersioningConfig, location=None, object_lock=Fals def get_latest_object(self, object_name): """returns the latest_object""" + if not self._objects[object_name]: + return None for version in self._objects[object_name]: obj = self._objects[object_name][version] if obj.is_latest: @@ -587,10 +589,13 @@ def _put_object( if object_name not in self.buckets[bucket_name].objects: self.buckets[bucket_name].objects[object_name] = {} else: - for old_version in self.buckets[bucket_name].objects[object_name]: - self.buckets[bucket_name].objects[object_name][ - old_version - ]._is_latest = False + latest_obj = self.buckets[bucket_name].get_latest_object(object_name) + if latest_obj: + latest_obj._is_latest = False + # for old_version in self.buckets[bucket_name].objects[object_name]: + # self.buckets[bucket_name].objects[object_name][ + # old_version + # ]._is_latest = False self.buckets[bucket_name].objects[object_name][version] = obj return obj @@ -977,12 +982,9 @@ def remove_object(self, bucket_name, object_name, version_id=None): if version_id: if version_id in self.buckets[bucket_name].objects[object_name]: del self.buckets[bucket_name].objects[object_name][version_id] - else: - # nothing to do - return else: del self.buckets[bucket_name].objects[object_name] - return + return except Exception: logging.error("remove_object(): Exception") logging.error(self.buckets) @@ -995,22 +997,16 @@ def remove_object(self, bucket_name, object_name, version_id=None): # version_id does not exist # nothing to do return - # elif ( - # self.buckets[bucket_name] - # .objects[object_name][version_id] - # .is_delete_marker - # == True - # ): - # de - # version_id exists but delete_market is already True - # nothing to do - # raise Exception("TODO") else: - # mark the object as deleted - # self.buckets[bucket_name].objects[object_name][ - # version_id - # ].is_delete_marker = True + latest_obj = self.buckets[bucket_name].get_latest_object( + object_name + ) del self.buckets[bucket_name].objects[object_name][version_id] + if version_id == latest_obj.version_id: + obj = list( + self.buckets[bucket_name].objects[object_name].values() + )[0] + obj._is_latest = True else: # version_id is False # @@ -1028,6 +1024,33 @@ def remove_object(self, bucket_name, object_name, version_id=None): length=0, is_delete_marker=True, ) + return + elif self.get_bucket_versioning(bucket_name).status == SUSPENDED: + if version_id: + latest_obj = self.buckets[bucket_name].get_latest_object( + object_name + ) + # latest_obj._is_latest = False + # obj = self.buckets[bucket_name].objects[version_id] + # obj._is_latest = True + # obj.is_delete_marker = True + latest_obj = self.buckets[bucket_name].get_latest_object( + object_name + ) + del self.buckets[bucket_name].objects[version_id] + if version_id == latest_obj.version_id: + obj = list( + self.buckets[bucket_name].objects[object_name].values() + )[0] + obj._is_latest = True + + else: + latest_obj = self.buckets[bucket_name].get_latest_object( + object_name + ) + latest_obj.is_delete_marker = True + else: + raise Exception("unexpected") except Exception as e: logging.error("remove_object(): Exception") diff --git a/tests/test_minio_mock.py b/tests/test_minio_mock.py index b7fe1df..208136a 100644 --- a/tests/test_minio_mock.py +++ b/tests/test_minio_mock.py @@ -125,25 +125,15 @@ def test_versioned_objects(minio_mock): client.remove_object(bucket_name, object_name) objects = list(client.list_objects(bucket_name, object_name, include_version=True)) - assert len(objects) == 5 - objects = list(client.list_objects(bucket_name, object_name)) - assert len(objects) == 0 - - client.fput_object(bucket_name, object_name, file_path) - objects = list(client.list_objects(bucket_name, object_name, include_version=True)) - assert len(objects) == 2 - last_version = objects[1].version_id + assert len(objects) == 5 - client.remove_object(bucket_name, object_name) - objects = list(client.list_objects(bucket_name, object_name, include_version=True)) - assert len(objects) == 3 - assert first_version == objects[0].version_id - assert last_version == objects[1].version_id - assert objects[2].is_delete_marker + with pytest.raises(S3Error) as error: + client.get_object(bucket_name, object_name, version_id=objects[3].version_id) + assert "not allowed against this resource" in str(error.value) with pytest.raises(S3Error) as error: - client.get_object(bucket_name, object_name, version_id=objects[2].version_id) + client.get_object(bucket_name, object_name, version_id=objects[4].version_id) assert "not allowed against this resource" in str(error.value) objects = list(client.list_objects(bucket_name, object_name)) @@ -177,12 +167,12 @@ def test_versioned_objects_after_upload(minio_mock): client.remove_object(bucket_name, object_name) objects = list(client.list_objects(bucket_name, object_name, include_version=True)) - assert len(objects) == 2 - assert objects[0].version_id == last_version - assert objects[1].version_id is None + assert len(objects) == 3 + assert objects[-1].is_delete_marker == True - assert not objects[0].is_delete_marker - assert objects[1].is_delete_marker + client.remove_object(bucket_name, object_name, "null") + objects = list(client.list_objects(bucket_name, object_name, include_version=True)) + assert len(objects) == 3 @pytest.mark.UNIT From 2902a47a9c1230d6e913f40274bda0bf87747431 Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sun, 21 Apr 2024 04:36:29 +0200 Subject: [PATCH 16/26] all tests passing --- pytest_minio_mock/plugin.py | 2 +- tests/test_minio_mock.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pytest_minio_mock/plugin.py b/pytest_minio_mock/plugin.py index e2642d0..5e4927e 100644 --- a/pytest_minio_mock/plugin.py +++ b/pytest_minio_mock/plugin.py @@ -1037,7 +1037,7 @@ def remove_object(self, bucket_name, object_name, version_id=None): latest_obj = self.buckets[bucket_name].get_latest_object( object_name ) - del self.buckets[bucket_name].objects[version_id] + del self.buckets[bucket_name].objects[object_name][version_id] if version_id == latest_obj.version_id: obj = list( self.buckets[bucket_name].objects[object_name].values() diff --git a/tests/test_minio_mock.py b/tests/test_minio_mock.py index 208136a..4edb011 100644 --- a/tests/test_minio_mock.py +++ b/tests/test_minio_mock.py @@ -165,14 +165,20 @@ def test_versioned_objects_after_upload(minio_mock): assert last_version is not None client.set_bucket_versioning(bucket_name, VersioningConfig(SUSPENDED)) + objects = list(client.list_objects(bucket_name, object_name, include_version=True)) + + client.remove_object(bucket_name, object_name, objects[0].version_id) + objects = list(client.list_objects(bucket_name, object_name, include_version=True)) + assert len(objects) == 2 + client.remove_object(bucket_name, object_name) objects = list(client.list_objects(bucket_name, object_name, include_version=True)) - assert len(objects) == 3 + assert len(objects) == 2 assert objects[-1].is_delete_marker == True client.remove_object(bucket_name, object_name, "null") objects = list(client.list_objects(bucket_name, object_name, include_version=True)) - assert len(objects) == 3 + assert len(objects) == 1 @pytest.mark.UNIT From cdf2116f6d0112addbd2328703492adbba356aad Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sun, 21 Apr 2024 04:59:40 +0200 Subject: [PATCH 17/26] cleanup --- pytest_minio_mock/plugin.py | 167 +++++++++++++++++++----------------- tests/test_minio_mock.py | 2 + 2 files changed, 91 insertions(+), 78 deletions(-) diff --git a/pytest_minio_mock/plugin.py b/pytest_minio_mock/plugin.py index 5e4927e..e4e39d4 100644 --- a/pytest_minio_mock/plugin.py +++ b/pytest_minio_mock/plugin.py @@ -112,8 +112,17 @@ def is_latest(self): """is this object marked latest""" return self._is_latest + @is_latest.setter + def is_latest(self, value): + """Set the value of is_latest""" + self._is_latest = value + class MockMinioBucket: + """ + Represents a mock Bucket in Minio storage. + """ + def __init__(self, versioning: VersioningConfig, location=None, object_lock=False): self._versioning = versioning self._objects = {} @@ -343,7 +352,7 @@ def fget_object( request. Defaults to None. sse (optional): Server-side encryption option. Defaults to None, as it's not used in the mock. - version_id (str, optional): The version ID of the object to download. Defaults to + version_id (str | None, optional): The version ID of the object to download. Defaults to None. extra_query_params (dict, optional): Additional query parameters for the request. Defaults to None. @@ -385,7 +394,7 @@ def get_object( which means the whole object. request_headers (dict, optional): Additional headers for the request. Defaults to None. ssec (optional): Server-side encryption option. Defaults to None. - version_id (str, optional): The version ID of the object. Defaults to None. + version_id (str | None, optional): The version ID of the object. Defaults to None. extra_query_params (dict, optional): Additional query parameters. Defaults to None. Returns: @@ -393,7 +402,7 @@ def get_object( """ self._health_check() try: - the_object = self.buckets[bucket_name].objects[object_name] + the_objects = self.buckets[bucket_name].objects[object_name] except KeyError: raise S3Error( message="The specified key does not exist.", @@ -418,23 +427,32 @@ def get_object( object_name=object_name, ) - # Do not check whether the bucket is versioned or not, because even - # if versioning is suspended, object could have had versions, and we can - # interact with them. And if versioning is Off, then None is a valid - # version too, and we can work with it - - # If version == None, try to find the first one that does not - # correspond to a deleted object. Note that it can still be None after - # that if versioning is Off, but that is not a problem. + # If version == None, give version_id a value if not version_id: + if not the_objects: + raise S3Error( + # code=NoSuchKey + message="The specified key does not exist.", + resource=f"/{bucket_name}/{object_name}", + request_id=None, + host_id=None, + response="mocked_response", + code=404, + bucket_name=bucket_name, + object_name=object_name, + ) + + # if versioning is disabled, look for the object with version_id = "null" if self.get_bucket_versioning(bucket_name).status == OFF: version_id = "null" + # if versioning is enabled or suspended, get the version_id of the latest_object else: latest_object = self.buckets[bucket_name].get_latest_object(object_name) - version_id = latest_object.version_id + if latest_object: + version_id = latest_object.version_id try: - the_object = the_object[version_id] + the_object = the_objects[version_id] except KeyError: raise S3Error( message="The specified version does not exist", @@ -536,6 +554,57 @@ def fput_object( part_size=part_size, ) + def put_object( + self, + bucket_name: str, + object_name: str, + data, + length: int, + content_type: str = "application/octet-stream", + metadata=None, + sse=None, + progress=None, + part_size: int = 0 + # num_parallel_uploads: int = 3, + # tags = None, + # retention = None, + # legal_hold: bool = False, + ): + """ + Simulates uploading an object to the mock Minio server. + + Stores an object in a specified bucket with the given data. + + Args: + bucket_name (str): The name of the bucket to upload to. + object_name (str): The object name to create in the bucket. + data: The data to upload. Can be bytes or a file-like object. + length (int): The length of the data to upload. + content_type (str, optional): The content type of the object. Defaults to + "application/octet-stream". + metadata (dict, optional): A dictionary of additional metadata for the object. Defaults + to None. + sse (optional): Server-side encryption option. Defaults to None. + progress (optional): Callback function to monitor progress. Defaults to None. + part_size (int, optional): The size of each part in multi-part upload. Defaults to 0. + + Returns: + str: Confirmation message indicating successful upload. + """ + obj = self._put_object( + bucket_name, + object_name, + data, + length, + content_type, + metadata, + sse, + progress, + part_size, + ) + + return "Upload successful" + def _put_object( self, bucket_name: str, @@ -591,65 +660,11 @@ def _put_object( else: latest_obj = self.buckets[bucket_name].get_latest_object(object_name) if latest_obj: - latest_obj._is_latest = False - # for old_version in self.buckets[bucket_name].objects[object_name]: - # self.buckets[bucket_name].objects[object_name][ - # old_version - # ]._is_latest = False + latest_obj.is_latest = False + self.buckets[bucket_name].objects[object_name][version] = obj return obj - def put_object( - self, - bucket_name: str, - object_name: str, - data, - length: int, - content_type: str = "application/octet-stream", - metadata=None, - sse=None, - progress=None, - part_size: int = 0 - # num_parallel_uploads: int = 3, - # tags = None, - # retention = None, - # legal_hold: bool = False, - ): - """ - Simulates uploading an object to the mock Minio server. - - Stores an object in a specified bucket with the given data. - - Args: - bucket_name (str): The name of the bucket to upload to. - object_name (str): The object name to create in the bucket. - data: The data to upload. Can be bytes or a file-like object. - length (int): The length of the data to upload. - content_type (str, optional): The content type of the object. Defaults to - "application/octet-stream". - metadata (dict, optional): A dictionary of additional metadata for the object. Defaults - to None. - sse (optional): Server-side encryption option. Defaults to None. - progress (optional): Callback function to monitor progress. Defaults to None. - part_size (int, optional): The size of each part in multi-part upload. Defaults to 0. - - Returns: - str: Confirmation message indicating successful upload. - """ - obj = self._put_object( - bucket_name, - object_name, - data, - length, - content_type, - metadata, - sse, - progress, - part_size, - ) - - return "Upload successful" - def get_presigned_url( self, method, @@ -676,7 +691,7 @@ def get_presigned_url( Defaults to 7 days. response_headers (dict, optional): Headers to include in the response. Defaults to None. request_date (datetime, optional): The date of the request. Defaults to None. - version_id (str, optional): The version ID of the object. Defaults to None. + version_id (str | None, optional): The version ID of the object. Defaults to None. extra_query_params (dict, optional): Additional query parameters to include in the presigned URL. Defaults to None. @@ -730,7 +745,7 @@ def presigned_get_object( Defaults to 7 days. response_headers (dict, optional): Headers to include in the response. Defaults to None. request_date (datetime, optional): The date of the request. Defaults to None. - version_id (str, optional): The version ID of the object. Defaults to None. + version_id (str | None, optional): The version ID of the object. Defaults to None. extra_query_params (dict, optional): Additional query parameters to include in the presigned URL. Defaults to None. @@ -968,7 +983,7 @@ def remove_object(self, bucket_name, object_name, version_id=None): Args: bucket_name (str): The name of the bucket. object_name (str): The name of the object to remove. - version_id (str, optional): The version to delete. + version_id (str | None, optional): The version to delete. Returns: None: The method has no return value but indicates successful removal. @@ -1006,7 +1021,7 @@ def remove_object(self, bucket_name, object_name, version_id=None): obj = list( self.buckets[bucket_name].objects[object_name].values() )[0] - obj._is_latest = True + obj.is_latest = True else: # version_id is False # @@ -1030,10 +1045,6 @@ def remove_object(self, bucket_name, object_name, version_id=None): latest_obj = self.buckets[bucket_name].get_latest_object( object_name ) - # latest_obj._is_latest = False - # obj = self.buckets[bucket_name].objects[version_id] - # obj._is_latest = True - # obj.is_delete_marker = True latest_obj = self.buckets[bucket_name].get_latest_object( object_name ) @@ -1042,7 +1053,7 @@ def remove_object(self, bucket_name, object_name, version_id=None): obj = list( self.buckets[bucket_name].objects[object_name].values() )[0] - obj._is_latest = True + obj.is_latest = True else: latest_obj = self.buckets[bucket_name].get_latest_object( diff --git a/tests/test_minio_mock.py b/tests/test_minio_mock.py index 4edb011..061baba 100644 --- a/tests/test_minio_mock.py +++ b/tests/test_minio_mock.py @@ -82,6 +82,8 @@ def test_versioned_objects(minio_mock): client.remove_object(bucket_name, object_name, version_id="null") objects = list(client.list_objects(bucket_name, object_name, include_version=True)) assert len(objects) == 0 + obj = client.get_object(bucket_name, object_name) + assert obj # Versioning Enabled client.set_bucket_versioning(bucket_name, VersioningConfig(ENABLED)) From d164f746bd00b0deb1090f6f17427f5719ae2a5c Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sun, 21 Apr 2024 05:01:54 +0200 Subject: [PATCH 18/26] all tests passing --- tests/test_minio_mock.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_minio_mock.py b/tests/test_minio_mock.py index 061baba..984d648 100644 --- a/tests/test_minio_mock.py +++ b/tests/test_minio_mock.py @@ -82,8 +82,10 @@ def test_versioned_objects(minio_mock): client.remove_object(bucket_name, object_name, version_id="null") objects = list(client.list_objects(bucket_name, object_name, include_version=True)) assert len(objects) == 0 - obj = client.get_object(bucket_name, object_name) - assert obj + + with pytest.raises(S3Error) as error: + _ = client.get_object(bucket_name, object_name) + assert "The specified key does not exist" in str(error.value) # Versioning Enabled client.set_bucket_versioning(bucket_name, VersioningConfig(ENABLED)) From ab72754c85895b6ae821c5abcbed6d04df90a49f Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sun, 21 Apr 2024 05:02:45 +0200 Subject: [PATCH 19/26] no pytest warnings --- tests/test_minio_mock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_minio_mock.py b/tests/test_minio_mock.py index 984d648..5137ed5 100644 --- a/tests/test_minio_mock.py +++ b/tests/test_minio_mock.py @@ -158,14 +158,14 @@ def test_versioned_objects_after_upload(minio_mock): objects = list(client.list_objects(bucket_name, object_name, include_version=True)) assert len(objects) == 1 first_version = objects[0].version_id - assert first_version is "null" + assert first_version == "null" client.fput_object(bucket_name, object_name, file_path) client.fput_object(bucket_name, object_name, file_path) objects = list(client.list_objects(bucket_name, object_name, include_version=True)) last_version = objects[1].version_id assert len(objects) == 3 - assert objects[-1].version_id is "null" + assert objects[-1].version_id == "null" assert last_version is not None client.set_bucket_versioning(bucket_name, VersioningConfig(SUSPENDED)) From b1072be27923dddfa7e706a91a1f2a7d6fb04cf3 Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sun, 21 Apr 2024 05:06:04 +0200 Subject: [PATCH 20/26] removed type str | None to support python3.8 and python3.9 --- pytest_minio_mock/plugin.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pytest_minio_mock/plugin.py b/pytest_minio_mock/plugin.py index e4e39d4..06afcc2 100644 --- a/pytest_minio_mock/plugin.py +++ b/pytest_minio_mock/plugin.py @@ -333,7 +333,7 @@ def fget_object( file_path, request_headers=None, sse=None, - version_id: str | None = None, + version_id=None, extra_query_params=None, ): """ @@ -376,7 +376,7 @@ def get_object( length: int = 0, request_headers=None, ssec=None, - version_id: str | None = None, + version_id=None, extra_query_params=None, ): """ @@ -673,7 +673,7 @@ def get_presigned_url( expires=datetime.timedelta(days=7), response_headers=None, request_date=None, - version_id: str | None = None, + version_id=None, extra_query_params=None, ): """ @@ -729,7 +729,7 @@ def presigned_get_object( expires=datetime.timedelta(days=7), response_headers=None, request_date=None, - version_id: str | None = None, + version_id=None, extra_query_params=None, ): """ From 82ecb2da99f53993d239511b9d5d80a67ffaa5ca Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sun, 21 Apr 2024 05:10:14 +0200 Subject: [PATCH 21/26] typo --- pytest_minio_mock/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest_minio_mock/plugin.py b/pytest_minio_mock/plugin.py index 06afcc2..b1d15b9 100644 --- a/pytest_minio_mock/plugin.py +++ b/pytest_minio_mock/plugin.py @@ -180,7 +180,7 @@ def base_url(self): return self._base_url @property - def bucket(self): + def buckets(self): """Get the dictionary of buckets in the server.""" return self._buckets From 57f17bc1f428a7bbcfaa89ac66addf839d6f3101 Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sun, 21 Apr 2024 05:15:53 +0200 Subject: [PATCH 22/26] changed version --- pytest_minio_mock/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pytest_minio_mock/__init__.py b/pytest_minio_mock/__init__.py index f2c0c67..70f4ad4 100644 --- a/pytest_minio_mock/__init__.py +++ b/pytest_minio_mock/__init__.py @@ -15,5 +15,5 @@ """ -__version__ = "0.1.10" +__version__ = "0.2.10" __author__ = "Oussama Jarrousse" diff --git a/setup.py b/setup.py index 5d0aca6..7b2d036 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ long_description=open("README.md").read(), keywords="pytest minio mock", extras_require={"dev": ["pre-commit", "tox"]}, - version="0.1.10", + version="0.2.10", long_description_content_type="text/markdown", classifiers=[ "Framework :: Pytest", From 03fcf54d7ea745f7ec9b2a0c15112ce0e78f0574 Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sun, 21 Apr 2024 05:22:21 +0200 Subject: [PATCH 23/26] linting and formatting --- pytest_minio_mock/plugin.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/pytest_minio_mock/plugin.py b/pytest_minio_mock/plugin.py index b1d15b9..8d20cd2 100644 --- a/pytest_minio_mock/plugin.py +++ b/pytest_minio_mock/plugin.py @@ -364,7 +364,14 @@ def fget_object( Returns: None: The method writes the object's data to a file and has no return value. """ - the_object = self.get_object(bucket_name, object_name, version_id=version_id) + the_object = self.get_object( + bucket_name, + object_name, + version_id=version_id, + request_headers=request_headers, + sse=sse, + extra_query_params=extra_query_params, + ) with open(file_path, "wb") as f: f.write(the_object.data) @@ -375,7 +382,7 @@ def get_object( offset: int = 0, length: int = 0, request_headers=None, - ssec=None, + sse=None, version_id=None, extra_query_params=None, ): @@ -393,7 +400,7 @@ def get_object( length (int, optional): The number of bytes of object data to retrieve. Defaults to 0, which means the whole object. request_headers (dict, optional): Additional headers for the request. Defaults to None. - ssec (optional): Server-side encryption option. Defaults to None. + sse (optional): Server-side encryption option. Defaults to None. version_id (str | None, optional): The version ID of the object. Defaults to None. extra_query_params (dict, optional): Additional query parameters. Defaults to None. @@ -403,7 +410,7 @@ def get_object( self._health_check() try: the_objects = self.buckets[bucket_name].objects[object_name] - except KeyError: + except KeyError as exc: raise S3Error( message="The specified key does not exist.", resource=f"/{bucket_name}/{object_name}", @@ -413,7 +420,7 @@ def get_object( code=404, bucket_name=bucket_name, object_name=object_name, - ) + ) from exc if version_id and version_id != "null" and not validators.uuid(version_id): raise S3Error( @@ -453,7 +460,7 @@ def get_object( try: the_object = the_objects[version_id] - except KeyError: + except KeyError as exc: raise S3Error( message="The specified version does not exist", resource=f"/{bucket_name}/{object_name}", @@ -463,7 +470,7 @@ def get_object( code=404, bucket_name=bucket_name, object_name=object_name, - ) + ) from exc if the_object.is_delete_marker: raise S3Error( @@ -591,7 +598,7 @@ def put_object( Returns: str: Confirmation message indicating successful upload. """ - obj = self._put_object( + _ = self._put_object( bucket_name, object_name, data, @@ -915,11 +922,14 @@ def _list_objects( yield Object( bucket_name=bucket_name, object_name=dir_name ) - # bucket_objects.append() - continue # Skip further processing to prevent adding the full object path - # Directly add the object for recursive listing or if it's a file in the current directory + # Skip further processing to prevent + # adding the full object path + continue + # Directly add the object for recursive listing + # or if it's a file in the current directory if include_version: - # Minio API always sort versions by time, it also includes delete markers at the end newwst first + # Minio API always sort versions by time, + # it also includes delete markers at the end newwst first versions_list = list( sorted( bucket_objects[obj_name].items(), From eeaebcc90cd469e9527458792f63442b20f9916a Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sun, 21 Apr 2024 15:28:34 +0200 Subject: [PATCH 24/26] refactoring; cleaning up; organising --- pytest_minio_mock/plugin.py | 463 +++++++++++++++++++++++------------- tests/test_minio_mock.py | 116 +++++++-- 2 files changed, 384 insertions(+), 195 deletions(-) diff --git a/pytest_minio_mock/plugin.py b/pytest_minio_mock/plugin.py index 8d20cd2..fb2c1b9 100644 --- a/pytest_minio_mock/plugin.py +++ b/pytest_minio_mock/plugin.py @@ -5,7 +5,9 @@ degree that is useful for testing purposes. Classes: + MockMinioObjectVersion: Represents a mock object version stored in Minio. MockMinioObject: Represents a mock object stored in Minio. + MockMinioBucket: Represents a mock bucket stored in Minio. MockMinioServer: Represents a single mock Minio server. MockMinioServers: Manages multiple mock Minio server instances. MockMinioClient: A mock version of the Minio client. @@ -43,9 +45,9 @@ # ObjectInfo = namedtuple("ObjectInfo", ["object_name"]) -class MockMinioObject: +class MockMinioObjectVersion: """ - Represents a mock object in Minio storage. + Represents a mock object version in Minio storage. Attributes: object_name (str): The name of the object. @@ -57,7 +59,7 @@ class MockMinioObject: def __init__(self, object_name, data, version_id, is_delete_marker, is_latest): """ - Initialize the MockMinioObject with a name and data. + Initialize the MockMinioObjectVersion with a name and data. Args: object_name (str): The name of the object. @@ -118,26 +120,194 @@ def is_latest(self, value): self._is_latest = value +class MockMinioObject: + """ + Represents a mock object in Minio storage. + """ + + def __init__(self, object_name): + self._object_name = object_name + self._versions = {} + + @property + def object_name(self): + """Get the name of the object stored at initialization""" + return self._object_name + + @property + def versions(self): + """Get the versions stored in the object""" + return self._versions + + def reset_latest(self): + """ + Sets the value of the latest flag to False in all object versions + """ + for _, obj in self._versions.items(): + if obj.is_latest: + obj.is_latest = False + return + + def get_latest(self): + """ + Returns: + The version that is marked latest. + If the object has no versions, it returns None. + if none of the versions is marked latest it raises an exception that indicates an implementation error + """ + if not self._versions: + return None + for _, obj in self._versions.items(): + if obj.is_latest: + return obj + raise RuntimeError("Implemnetation Error") + + def put_object_version(self, version_id, obj): + """ + Inserts a object version in the _versions map. + """ + self.reset_latest() + obj.latest = True + self._versions[version_id] = obj + + def put_object( + self, + object_name: str, + data, + length: int, + content_type: str = "application/octet-stream", + metadata=None, + sse=None, + progress=None, + part_size: int = 0, + # num_parallel_uploads: int = 3, + # tags = None, + # retention = None, + # legal_hold: bool = False, + versioning: VersioningConfig = VersioningConfig(), + ): + """ + Returns: + the put object + """ + + # According to + # https://min.io/docs/minio/linux/administration/object-management/object-versioning.html#suspend-bucket-versioning + # objects created when versioning is suspended have a 'null' version ID (None in Python) + + version_id = "null" + # If status is enabled, create a new version UUID + if versioning.status == ENABLED: + version_id = str(uuid4()) + + obj = MockMinioObjectVersion( + object_name=object_name, + data=data, + version_id=version_id, + is_delete_marker=False, + is_latest=True, + ) + # If versioning is OFF, there can only be one version of an object (store a read version_id non-the-less) + if versioning.status == OFF: + self._versions = {} + + self.put_object_version(version_id, obj) + return obj + + def get_object(self, version_id, versioning: VersioningConfig): + """ + Returns + the stored object if versioning is disabled + the latest version of the object if versioning is enabled + """ + if version_id and version_id != "null" and not validators.uuid(version_id): + raise S3Error( + message="Invalid version id specified", + resource=f"/{self.bucket_name}/{object_name}", + request_id=None, + host_id=None, + response="mocked_response", + code=422, + bucket_name=self.bucket_name, + object_name=object_name, + ) + + # If version == None, give version_id a value + if not version_id: + if not self._versions: + raise S3Error( + # code=NoSuchKey + message="The specified key does not exist.", + resource="", + request_id=None, + host_id=None, + response="mocked_response", + code=404, + bucket_name=None, + object_name=self.object_name, + ) + + # if versioning is disabled, look for the object with version_id = "null" + if versioning.status == OFF: + version_id = "null" + # if versioning is enabled or suspended, get the version_id of the latest_version + else: + latest_object_version = self.get_latest() + if latest_object_version: + version_id = latest_object_version.version_id + + # now try to get the object with that version_id + try: + the_object_version = self._versions[version_id] + except KeyError as exc: + raise S3Error( + message="The specified version does not exist", + resource="", + request_id=None, + host_id=None, + response="mocked_response", + code=404, + bucket_name=None, + object_name=self.object_name, + ) from exc + + # if the delete_marker is set raise an error + if the_object_version.is_delete_marker: + raise S3Error( + message="The specified method is not allowed against this resource.", + resource="", + request_id=None, + host_id=None, + response="mocked_response", + code=403, + bucket_name=None, + object_name=self.object_name, + ) + return the_object_version + + class MockMinioBucket: """ Represents a mock Bucket in Minio storage. """ - def __init__(self, versioning: VersioningConfig, location=None, object_lock=False): + def __init__( + self, + bucket_name, + versioning: VersioningConfig, + location=None, + object_lock=False, + ): + self._bucket_name = bucket_name self._versioning = versioning self._objects = {} self._location = location self._object_lock = object_lock - def get_latest_object(self, object_name): - """returns the latest_object""" - if not self._objects[object_name]: - return None - for version in self._objects[object_name]: - obj = self._objects[object_name][version] - if obj.is_latest: - return obj - raise Exception("no latest") + @property + def bucket_name(self): + """Gets the name of the bucket stored at initialization.""" + return self._bucket_name @property def objects(self): @@ -154,6 +324,72 @@ def versioning(self, versioning: VersioningConfig): """Set the versioning config of the bucket.""" self._versioning = versioning + def put_object( + self, + object_name: str, + data, + length: int, + content_type: str = "application/octet-stream", + metadata=None, + sse=None, + progress=None, + part_size: int = 0, + # num_parallel_uploads: int = 3, + # tags = None, + # retention = None, + # legal_hold: bool = False, + ): + if object_name not in self.objects: + self.objects[object_name] = MockMinioObject(object_name) + + obj = self.objects[object_name].put_object( + object_name=object_name, + data=data, + length=length, + content_type=content_type, + metadata=metadata, + sse=sse, + progress=progress, + part_size=part_size, + # num_parallel_uploads: int = 3, + # tags = None, + # retention = None, + # legal_hold: bool = False, + versioning=self.versioning, + ) + + return obj + + def get_object(self, object_name, version_id): + try: + the_object = self.objects[object_name] + except KeyError as exc: + raise S3Error( + message="The specified key does not exist.", + resource=f"/{self.bucket_name}/{object_name}", + request_id=None, + host_id=None, + response="mocked_response", + code=404, + bucket_name=self.bucket_name, + object_name=object_name, + ) from exc + + try: + the_object_version = the_object.get_object(version_id, self.versioning) + except S3Error as e: + raise S3Error( + message=e.message, + response=e.response, + resource=f"/{self.bucket_name}/{object_name}", + host_id=e._host_id, + request_id=e.request_id, + code=e.code, + bucket_name=self.bucket_name, + object_name=object_name, + ) + return the_object_version + class MockMinioServer: """ @@ -194,29 +430,29 @@ def __len__(self): return len(self._buckets) def keys(self): - """Returns the keys of the self._buckets dictionary""" + """Returns the keys of the self._buckets dictionary.""" return self._buckets.keys() def values(self): - """Returns the values of the self._buckets dictionary""" + """Returns the values of the self._buckets dictionary.""" return self._buckets.values() def items(self): - """Returns the items of the self._buckets dictionary""" + """Returns the items of the self._buckets dictionary.""" return self._buckets.items() def get(self, key, default=None): """get a specific bucket, - or default if key is not in the self._buckets dictionary + or default if key is not in the self._buckets dictionary. """ return self._buckets.get(key, default) def pop(self, key, default=None): - """pops a specific bucket""" + """pops a specific bucket.""" return self._buckets.pop(key, default) if key in self._buckets else default def update(self, other): - """updates the self._buckets dictionary with another dictionary""" + """updates the self._buckets dictionary with another dictionary.""" self._buckets.update(other) def __contains__(self, item): @@ -408,82 +644,14 @@ def get_object( HTTPResponse: A response object containing the object data. """ self._health_check() - try: - the_objects = self.buckets[bucket_name].objects[object_name] - except KeyError as exc: - raise S3Error( - message="The specified key does not exist.", - resource=f"/{bucket_name}/{object_name}", - request_id=None, - host_id=None, - response="mocked_response", - code=404, - bucket_name=bucket_name, - object_name=object_name, - ) from exc - if version_id and version_id != "null" and not validators.uuid(version_id): - raise S3Error( - message="Invalid version id specified", - resource=f"/{bucket_name}/{object_name}", - request_id=None, - host_id=None, - response="mocked_response", - code=422, - bucket_name=bucket_name, - object_name=object_name, - ) - - # If version == None, give version_id a value - if not version_id: - if not the_objects: - raise S3Error( - # code=NoSuchKey - message="The specified key does not exist.", - resource=f"/{bucket_name}/{object_name}", - request_id=None, - host_id=None, - response="mocked_response", - code=404, - bucket_name=bucket_name, - object_name=object_name, - ) - - # if versioning is disabled, look for the object with version_id = "null" - if self.get_bucket_versioning(bucket_name).status == OFF: - version_id = "null" - # if versioning is enabled or suspended, get the version_id of the latest_object - else: - latest_object = self.buckets[bucket_name].get_latest_object(object_name) - if latest_object: - version_id = latest_object.version_id - - try: - the_object = the_objects[version_id] - except KeyError as exc: - raise S3Error( - message="The specified version does not exist", - resource=f"/{bucket_name}/{object_name}", - request_id=None, - host_id=None, - response="mocked_response", - code=404, - bucket_name=bucket_name, - object_name=object_name, - ) from exc + the_object_version = self.buckets[bucket_name].get_object( + object_name, version_id + ) + if not the_object_version: + raise RuntimeError("Implementation Error") - if the_object.is_delete_marker: - raise S3Error( - message="The specified method is not allowed against this resource.", - resource=f"/{bucket_name}/{object_name}", - request_id=None, - host_id=None, - response="mocked_response", - code=403, - bucket_name=bucket_name, - object_name=object_name, - ) - data = the_object.data + data = the_object_version.data # Create a buffer containing the data if isinstance(data, io.BytesIO): @@ -571,7 +739,7 @@ def put_object( metadata=None, sse=None, progress=None, - part_size: int = 0 + part_size: int = 0, # num_parallel_uploads: int = 3, # tags = None, # retention = None, @@ -598,37 +766,7 @@ def put_object( Returns: str: Confirmation message indicating successful upload. """ - _ = self._put_object( - bucket_name, - object_name, - data, - length, - content_type, - metadata, - sse, - progress, - part_size, - ) - return "Upload successful" - - def _put_object( - self, - bucket_name: str, - object_name: str, - data, - length: int, - content_type: str = "application/octet-stream", - metadata=None, - sse=None, - progress=None, - part_size: int = 0, - # num_parallel_uploads: int = 3, - # tags = None, - # retention = None, - # legal_hold: bool = False, - is_delete_marker=False, - ): self._health_check() if not self.bucket_exists(bucket_name): raise S3Error( @@ -642,35 +780,22 @@ def _put_object( object_name=None, ) - # According to - # https://min.io/docs/minio/linux/administration/object-management/object-versioning.html#suspend-bucket-versioning - # objects created when versioning is suspended have a 'null' version ID (None in Python) - - version = "null" - # If status is enabled, create a new version UUID - if self.get_bucket_versioning(bucket_name).status == ENABLED: - version = str(uuid4()) - - obj = MockMinioObject( + _ = self.buckets[bucket_name].put_object( object_name=object_name, data=data, - version_id=version, - is_delete_marker=is_delete_marker, - is_latest=True, + length=length, + content_type=content_type, + metadata=metadata, + sse=sse, + progress=progress, + part_size=part_size, + # num_parallel_uploads: int = 3, + # tags = None, + # retention = None, + # legal_hold: bool = False, ) - # If versioning is OFF, there can only be one version of an object (store a read version_id non-the-less) - if self.get_bucket_versioning(bucket_name).status == OFF: - self.buckets[bucket_name].objects[object_name] = {version: obj} - else: - if object_name not in self.buckets[bucket_name].objects: - self.buckets[bucket_name].objects[object_name] = {} - else: - latest_obj = self.buckets[bucket_name].get_latest_object(object_name) - if latest_obj: - latest_obj.is_latest = False - self.buckets[bucket_name].objects[object_name][version] = obj - return obj + return "Upload successful" def get_presigned_url( self, @@ -827,7 +952,10 @@ def make_bucket(self, bucket_name, location=None, object_lock=False): """ self._health_check() self.buckets[bucket_name] = MockMinioBucket( - versioning=VersioningConfig(), location=location, object_lock=object_lock + bucket_name=bucket_name, + versioning=VersioningConfig(), + location=location, + object_lock=object_lock, ) return True @@ -907,13 +1035,13 @@ def _list_objects( # bucket_objects = [] seen_prefixes = set() - for obj_name in bucket_objects.keys(): - if obj_name.startswith(prefix) and ( - start_after == "" or obj_name > start_after + for object_name in bucket_objects.keys(): + if object_name.startswith(prefix) and ( + start_after == "" or object_name > start_after ): # Handle non-recursive listing by identifying and adding unique directory names if not recursive: - sub_path = obj_name[len(prefix) :].strip("/") + sub_path = object_name[len(prefix) :].strip("/") dir_end_idx = sub_path.find("/") if dir_end_idx != -1: dir_name = prefix + sub_path[: dir_end_idx + 1] @@ -932,7 +1060,7 @@ def _list_objects( # it also includes delete markers at the end newwst first versions_list = list( sorted( - bucket_objects[obj_name].items(), + bucket_objects[object_name].items(), key=lambda i: ( i[1].is_delete_marker, -i[1].last_modified.timestamp(), @@ -942,7 +1070,7 @@ def _list_objects( for version, obj in versions_list: yield Object( bucket_name=bucket_name, - object_name=obj_name, + object_name=object_name, last_modified=obj.last_modified, version_id=version, is_latest="true" if obj.is_latest else "false", @@ -950,11 +1078,11 @@ def _list_objects( ) else: # only yield if the object is not a delete marker - obj = buckets[bucket_name].get_latest_object(obj_name) + obj = buckets[bucket_name].objects[object_name].get_latest() if not obj.is_delete_marker: yield Object( bucket_name=bucket_name, - object_name=obj_name, + object_name=object_name, last_modified=obj.last_modified, version_id=None, is_latest=None, @@ -1023,8 +1151,8 @@ def remove_object(self, bucket_name, object_name, version_id=None): # nothing to do return else: - latest_obj = self.buckets[bucket_name].get_latest_object( - object_name + latest_obj = ( + self.buckets[bucket_name].objects[object_name].get_latest() ) del self.buckets[bucket_name].objects[object_name][version_id] if version_id == latest_obj.version_id: @@ -1035,8 +1163,8 @@ def remove_object(self, bucket_name, object_name, version_id=None): else: # version_id is False # - latest_obj = self.buckets[bucket_name].get_latest_object( - object_name + latest_obj = ( + self.buckets[bucket_name].objects[object_name].get_latest() ) if latest_obj.is_delete_marker: # nothing to do @@ -1052,11 +1180,8 @@ def remove_object(self, bucket_name, object_name, version_id=None): return elif self.get_bucket_versioning(bucket_name).status == SUSPENDED: if version_id: - latest_obj = self.buckets[bucket_name].get_latest_object( - object_name - ) - latest_obj = self.buckets[bucket_name].get_latest_object( - object_name + latest_obj = ( + self.buckets[bucket_name].objects[object_name].get_latest() ) del self.buckets[bucket_name].objects[object_name][version_id] if version_id == latest_obj.version_id: @@ -1066,8 +1191,8 @@ def remove_object(self, bucket_name, object_name, version_id=None): obj.is_latest = True else: - latest_obj = self.buckets[bucket_name].get_latest_object( - object_name + latest_obj = ( + self.buckets[bucket_name].objects[object_name].get_latest() ) latest_obj.is_delete_marker = True else: diff --git a/tests/test_minio_mock.py b/tests/test_minio_mock.py index 5137ed5..7b0866e 100644 --- a/tests/test_minio_mock.py +++ b/tests/test_minio_mock.py @@ -10,24 +10,40 @@ from minio.versioningconfig import VersioningConfig from pytest_minio_mock.plugin import MockMinioBucket +from pytest_minio_mock.plugin import MockMinioObject + + +@pytest.mark.UNIT +class TestsMockMinioObject: + @pytest.mark.UNIT + def test_mock_minio_object_init(self): + mock_minio_object = MockMinioObject("test-object") + assert mock_minio_object.versions == {} @pytest.mark.UNIT class TestsMockMinioBucket: @pytest.mark.UNIT - def test_init(self): - mock_minio_bucket = MockMinioBucket(None) - assert mock_minio_bucket._versioning == None - assert mock_minio_bucket._objects == {} + def test_mock_minio_bucket_init(self): + mock_minio_bucket = MockMinioBucket( + bucket_name="test-bucket", versioning=VersioningConfig() + ) + assert mock_minio_bucket.bucket_name == "test-bucket" + assert mock_minio_bucket.versioning.status == OFF + assert mock_minio_bucket.objects == {} - versioning_config = VersioningConfig() - mock_minio_bucket = MockMinioBucket(versioning_config) + versioning_config = VersioningConfig(ENABLED) + mock_minio_bucket = MockMinioBucket( + bucket_name="test-bucket", versioning=versioning_config + ) assert isinstance(mock_minio_bucket._versioning, VersioningConfig) - assert mock_minio_bucket.versioning.status == OFF + assert mock_minio_bucket.versioning.status == ENABLED @pytest.mark.UNIT def test_versioning(self): - mock_minio_bucket = MockMinioBucket(VersioningConfig()) + mock_minio_bucket = MockMinioBucket( + bucket_name="test-bucket", versioning=VersioningConfig() + ) versioning_config = mock_minio_bucket.versioning assert isinstance(versioning_config, VersioningConfig) assert versioning_config.status == OFF @@ -47,9 +63,9 @@ def test_make_bucket(minio_mock): assert client.bucket_exists(bucket_name), "Bucket should exist after creation" -@pytest.mark.UNIT @pytest.mark.API -def test_adding_and_removing_objects_basic(minio_mock): +@pytest.mark.FUNC +def test_putting_and_removing_objects_no_versionning(minio_mock): # simple thing bucket_name = "test-bucket" object_name = "test-object" @@ -62,40 +78,61 @@ def test_adding_and_removing_objects_basic(minio_mock): assert ( object_name in client.buckets[bucket_name].objects ), "Object should be in the bucket after upload" + objects = list(client.list_objects(bucket_name)) + assert len(objects) == 1 client.remove_object(bucket_name, object_name) assert object_name not in client.buckets[bucket_name].objects + objects = list(client.list_objects(bucket_name)) + assert len(objects) == 0 + + # even if include version is True nothing should change because versioning is OFF + objects = list(client.list_objects(bucket_name, include_version=True)) + assert len(objects) == 0 + + # test retrieving object after it has been removed + with pytest.raises(S3Error) as error: + _ = client.get_object(bucket_name, object_name) + assert "The specified key does not exist" in str(error.value) -@pytest.mark.UNIT @pytest.mark.API -def test_versioned_objects(minio_mock): +@pytest.mark.FUNC +def test_putting_objects_with_versionning_enabled(minio_mock): + client = Minio("http://local.host:9000") bucket_name = "test-bucket" object_name = "test-object" file_path = "tests/fixtures/maya.jpeg" - - client = Minio("http://local.host:9000") client.make_bucket(bucket_name) - + # Versioning Enabled + client.set_bucket_versioning(bucket_name, VersioningConfig(ENABLED)) + # Add two objects client.fput_object(bucket_name, object_name, file_path) - objects = list(client.list_objects(bucket_name, object_name, include_version=True)) + client.fput_object(bucket_name, object_name, file_path) + # they should be two versions of the same object + # check list_objects with include_version=False returns only one object with is_latest=True + objects = list(client.list_objects(bucket_name, object_name, include_version=False)) assert len(objects) == 1 - client.remove_object(bucket_name, object_name, version_id="null") + # check that versions are stored correctly and retrieved correctly objects = list(client.list_objects(bucket_name, object_name, include_version=True)) - assert len(objects) == 0 + assert len(objects) == 2 - with pytest.raises(S3Error) as error: - _ = client.get_object(bucket_name, object_name) - assert "The specified key does not exist" in str(error.value) + +@pytest.mark.API +@pytest.mark.FUNC +def test_removing_object_version_with_versionning_enabled(minio_mock): + client = Minio("http://local.host:9000") + bucket_name = "test-bucket" + object_name = "test-object" + file_path = "tests/fixtures/maya.jpeg" + client.make_bucket(bucket_name) # Versioning Enabled client.set_bucket_versioning(bucket_name, VersioningConfig(ENABLED)) # Add two objects client.fput_object(bucket_name, object_name, file_path) client.fput_object(bucket_name, object_name, file_path) - # list_objects should sort by newest - objects = list(client.list_objects(bucket_name, object_name, include_version=True)) - assert len(objects) == 2 + objects = list(client.list_objects(bucket_name, object_name, include_version=True)) first_version = objects[0].version_id last_version = objects[1].version_id @@ -103,6 +140,7 @@ def test_versioned_objects(minio_mock): objects = list(client.list_objects(bucket_name, object_name, include_version=True)) assert len(objects) == 1 assert objects[0].version_id == last_version + assert objects[0].is_latest is True client.fput_object(bucket_name, object_name, file_path) objects = list(client.list_objects(bucket_name, object_name, include_version=True)) @@ -110,16 +148,39 @@ def test_versioned_objects(minio_mock): first_version = objects[0].version_id assert first_version != last_version + +@pytest.mark.API +@pytest.mark.FUNC +def test_putting_and_removing_and_listing_bjecst_with_versionning_enabled(minio_mock): + client = Minio("http://local.host:9000") + bucket_name = "test-bucket" + object_name = "test-object" + file_path = "tests/fixtures/maya.jpeg" + client.make_bucket(bucket_name) + + # Versioning Enabled + client.set_bucket_versioning(bucket_name, VersioningConfig(ENABLED)) + # Add two objects + client.fput_object(bucket_name, object_name, file_path) + client.fput_object(bucket_name, object_name, file_path) + objects = list(client.list_objects(bucket_name, object_name, include_version=True)) + assert len(objects) == 2 + # removing the object with versioning enabled will add a delete marker client.remove_object(bucket_name, object_name) objects = list(client.list_objects(bucket_name, object_name, include_version=True)) assert len(objects) == 3 + assert objects[-1].is_delete_marker == True + + # removing the object again will have no effect client.remove_object(bucket_name, object_name) objects = list(client.list_objects(bucket_name, object_name, include_version=True)) assert len(objects) == 3 + # listing an object marked for deletion will return an empty list objects = list(client.list_objects(bucket_name, object_name)) assert len(objects) == 0 + # putting a new version after deletion will add a new version client.fput_object(bucket_name, object_name, file_path) objects = list(client.list_objects(bucket_name, object_name, include_version=True)) assert len(objects) == 4 @@ -127,15 +188,17 @@ def test_versioned_objects(minio_mock): objects = list(client.list_objects(bucket_name, object_name)) assert len(objects) == 1 + # removing the object again with versioning enabled will add a new deletion marker client.remove_object(bucket_name, object_name) objects = list(client.list_objects(bucket_name, object_name, include_version=True)) - assert len(objects) == 5 + # trying to an object marked for deletion by version will raise an exception with pytest.raises(S3Error) as error: client.get_object(bucket_name, object_name, version_id=objects[3].version_id) assert "not allowed against this resource" in str(error.value) + # trying to an object marked for deletion by version will raise an exception with pytest.raises(S3Error) as error: client.get_object(bucket_name, object_name, version_id=objects[4].version_id) assert "not allowed against this resource" in str(error.value) @@ -144,8 +207,8 @@ def test_versioned_objects(minio_mock): assert len(objects) == 0 -@pytest.mark.UNIT @pytest.mark.API +@pytest.mark.FUNC def test_versioned_objects_after_upload(minio_mock): bucket_name = "test-bucket" object_name = "test-object" @@ -187,6 +250,7 @@ def test_versioned_objects_after_upload(minio_mock): @pytest.mark.UNIT @pytest.mark.API +@pytest.mark.FUNC @pytest.mark.parametrize("versioned", (True, False)) def test_file_download(minio_mock, versioned): bucket_name = "test-bucket" From cbe43f4042d325b2862db7d82ae0d5613214d23b Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sun, 21 Apr 2024 18:59:31 +0200 Subject: [PATCH 25/26] refactoring client.remove_object() --- pytest_minio_mock/plugin.py | 293 ++++++++++++++++++------------------ 1 file changed, 147 insertions(+), 146 deletions(-) diff --git a/pytest_minio_mock/plugin.py b/pytest_minio_mock/plugin.py index fb2c1b9..10d1533 100644 --- a/pytest_minio_mock/plugin.py +++ b/pytest_minio_mock/plugin.py @@ -285,6 +285,18 @@ def get_object(self, version_id, versioning: VersioningConfig): ) return the_object_version + def list_versions(self): + versions_list = list( + sorted( + self._versions.items(), + key=lambda i: ( + i[1].is_delete_marker, + -i[1].last_modified.timestamp(), + ), + ) + ) + return versions_list + class MockMinioBucket: """ @@ -339,6 +351,10 @@ def put_object( # retention = None, # legal_hold: bool = False, ): + """ + Returns + newly added object + """ if object_name not in self.objects: self.objects[object_name] = MockMinioObject(object_name) @@ -360,6 +376,73 @@ def put_object( return obj + def remove_object(self, object_name, version_id=None): + """ """ + if object_name not in self.objects: + # object does not exist, so nothing to do + return + try: + if self.versioning.status == OFF: + if version_id: + if version_id in self.objects[object_name]: + del self.objects[object_name][version_id] + else: + del self.objects[object_name] + return + except Exception as e: + logging.error("remove_object(): Exception") + logging.error(e) + raise e + + try: + if self.versioning.status == ENABLED: + if version_id: + if version_id not in self.objects[object_name]: + # version_id does not exist + # nothing to do + return + else: + latest_obj = self.objects[object_name].get_latest() + del self.objects[object_name][version_id] + if version_id == latest_obj.version_id: + obj = list(self.objects[object_name].values())[0] + obj.is_latest = True + + else: # version_id is False + # + latest_obj = self.objects[object_name].get_latest() + if latest_obj.is_delete_marker: + # nothing to do + return + + self._put_object( + bucket_name=bucket_name, + object_name=object_name, + data=b"", + length=0, + is_delete_marker=True, + ) + return + elif self.versioning.status == SUSPENDED: + if version_id: + latest_obj = self.objects[object_name].get_latest() + del self.objects[object_name][version_id] + if version_id == latest_obj.version_id: + obj = list(self.objects[object_name].values())[0] + obj.is_latest = True + + else: + latest_obj = self.objects[object_name].get_latest() + latest_obj.is_delete_marker = True + else: + raise Exception("unexpected") + + except Exception as e: + logging.error("remove_object(): Exception") + logging.error(self.buckets) + logging.error(e) + raise + def get_object(self, object_name, version_id): try: the_object = self.objects[object_name] @@ -390,6 +473,67 @@ def get_object(self, object_name, version_id): ) return the_object_version + def list_objects( + self, + prefix="", + recursive=False, + start_after="", + include_version=False, + ): + """ + Returns + Iterator of MockMinioObjectVersions of the current bucket + """ + # Initialization + # bucket_objects = [] + seen_prefixes = set() + + for object_name, obj in self.objects.items(): + if object_name.startswith(prefix) and ( + start_after == "" or object_name > start_after + ): + # Handle non-recursive listing by identifying and adding unique directory names + if not recursive: + sub_path = object_name[len(prefix) :].strip("/") + dir_end_idx = sub_path.find("/") + if dir_end_idx != -1: + dir_name = prefix + sub_path[: dir_end_idx + 1] + if dir_name not in seen_prefixes: + seen_prefixes.add(dir_name) + yield Object( + bucket_name=self.bucket_name, object_name=dir_name + ) + # Skip further processing to prevent + # adding the full object path + continue + # Directly add the object for recursive listing + # or if it's a file in the current directory + if include_version: + # Minio API always sort versions by time, + # it also includes delete markers at the end newwst first + versions_list = obj.list_versions() + for version, obj_version in versions_list: + yield Object( + bucket_name=self.bucket_name, + object_name=object_name, + last_modified=obj_version.last_modified, + version_id=version, + is_latest="true" if obj_version.is_latest else "false", + is_delete_marker=obj_version.is_delete_marker, + ) + else: + obj_version = obj.get_latest() + # only yield if the object is not a delete marker + if not obj_version.is_delete_marker: + yield Object( + bucket_name=self.bucket_name, + object_name=object_name, + last_modified=obj_version.last_modified, + version_id=None, + is_latest=None, + is_delete_marker=obj_version.is_delete_marker, + ) + class MockMinioServer: """ @@ -1022,73 +1166,6 @@ def list_objects( that match the specified conditions. """ - def _list_objects( - buckets, - bucket_name, - prefix="", - recursive=False, - start_after="", - include_version=False, - ): - # Initialization - bucket_objects = buckets[bucket_name].objects - # bucket_objects = [] - seen_prefixes = set() - - for object_name in bucket_objects.keys(): - if object_name.startswith(prefix) and ( - start_after == "" or object_name > start_after - ): - # Handle non-recursive listing by identifying and adding unique directory names - if not recursive: - sub_path = object_name[len(prefix) :].strip("/") - dir_end_idx = sub_path.find("/") - if dir_end_idx != -1: - dir_name = prefix + sub_path[: dir_end_idx + 1] - if dir_name not in seen_prefixes: - seen_prefixes.add(dir_name) - yield Object( - bucket_name=bucket_name, object_name=dir_name - ) - # Skip further processing to prevent - # adding the full object path - continue - # Directly add the object for recursive listing - # or if it's a file in the current directory - if include_version: - # Minio API always sort versions by time, - # it also includes delete markers at the end newwst first - versions_list = list( - sorted( - bucket_objects[object_name].items(), - key=lambda i: ( - i[1].is_delete_marker, - -i[1].last_modified.timestamp(), - ), - ) - ) - for version, obj in versions_list: - yield Object( - bucket_name=bucket_name, - object_name=object_name, - last_modified=obj.last_modified, - version_id=version, - is_latest="true" if obj.is_latest else "false", - is_delete_marker=obj.is_delete_marker, - ) - else: - # only yield if the object is not a delete marker - obj = buckets[bucket_name].objects[object_name].get_latest() - if not obj.is_delete_marker: - yield Object( - bucket_name=bucket_name, - object_name=object_name, - last_modified=obj.last_modified, - version_id=None, - is_latest=None, - is_delete_marker=obj.is_delete_marker, - ) - try: if bucket_name not in self.buckets: raise S3Error( @@ -1101,9 +1178,8 @@ def _list_objects( bucket_name=bucket_name, object_name=None, ) - return _list_objects( - self.buckets, - bucket_name, + return self.buckets[bucket_name].list_objects( + # self.buckets, prefix, recursive, start_after, @@ -1127,82 +1203,7 @@ def remove_object(self, bucket_name, object_name, version_id=None): None: The method has no return value but indicates successful removal. """ self._health_check() - if object_name not in self.buckets[bucket_name].objects: - # object does not exist: nothing to do - return - try: - if self.get_bucket_versioning(bucket_name).status == OFF: - if version_id: - if version_id in self.buckets[bucket_name].objects[object_name]: - del self.buckets[bucket_name].objects[object_name][version_id] - else: - del self.buckets[bucket_name].objects[object_name] - return - except Exception: - logging.error("remove_object(): Exception") - logging.error(self.buckets) - raise - - try: - if self.get_bucket_versioning(bucket_name).status == ENABLED: - if version_id: - if version_id not in self.buckets[bucket_name].objects[object_name]: - # version_id does not exist - # nothing to do - return - else: - latest_obj = ( - self.buckets[bucket_name].objects[object_name].get_latest() - ) - del self.buckets[bucket_name].objects[object_name][version_id] - if version_id == latest_obj.version_id: - obj = list( - self.buckets[bucket_name].objects[object_name].values() - )[0] - obj.is_latest = True - - else: # version_id is False - # - latest_obj = ( - self.buckets[bucket_name].objects[object_name].get_latest() - ) - if latest_obj.is_delete_marker: - # nothing to do - return - - self._put_object( - bucket_name=bucket_name, - object_name=object_name, - data=b"", - length=0, - is_delete_marker=True, - ) - return - elif self.get_bucket_versioning(bucket_name).status == SUSPENDED: - if version_id: - latest_obj = ( - self.buckets[bucket_name].objects[object_name].get_latest() - ) - del self.buckets[bucket_name].objects[object_name][version_id] - if version_id == latest_obj.version_id: - obj = list( - self.buckets[bucket_name].objects[object_name].values() - )[0] - obj.is_latest = True - - else: - latest_obj = ( - self.buckets[bucket_name].objects[object_name].get_latest() - ) - latest_obj.is_delete_marker = True - else: - raise Exception("unexpected") - - except Exception as e: - logging.error("remove_object(): Exception") - logging.error(self.buckets) - logging.error(e) - raise + return self.buckets[bucket_name].remove_object(object_name, version_id=None) @pytest.fixture From f44965fc1cff0ecbf12c417262223f3f6c614b6f Mon Sep 17 00:00:00 2001 From: Oussama Jarrousse Date: Sun, 21 Apr 2024 21:13:31 +0200 Subject: [PATCH 26/26] refactoring complete; all tests are passing --- pytest_minio_mock/plugin.py | 126 ++++++++++++++++++++---------------- tests/test_minio_mock.py | 2 +- 2 files changed, 72 insertions(+), 56 deletions(-) diff --git a/pytest_minio_mock/plugin.py b/pytest_minio_mock/plugin.py index 10d1533..6984f15 100644 --- a/pytest_minio_mock/plugin.py +++ b/pytest_minio_mock/plugin.py @@ -297,6 +297,70 @@ def list_versions(self): ) return versions_list + def remove_object(self, version_id, versioning: VersioningConfig): + """ + Returns + nothing + """ + try: + if versioning.status == OFF: + if version_id: + if version_id in self.versions: + del self.versions[version_id] + else: + raise RuntimeError("This should not happen") + return + except Exception as e: + logging.error("remove_object(): Exception") + logging.error(e) + raise e + try: + if versioning.status == ENABLED: + if version_id: + if version_id not in self.versions: + # version_id does not exist + # nothing to do + return + else: + latest_obj = self.get_latest() + del self.versions[version_id] + if version_id == latest_obj.version_id: + obj = list(self.versions.values())[0] + obj.is_latest = True + else: # version_id is False + latest_obj = self.get_latest() + if latest_obj.is_delete_marker: + # nothing to do + return + + version_id = str(uuid4()) + + obj = MockMinioObjectVersion( + object_name=self.object_name, + data=b"", + version_id=version_id, + is_delete_marker=True, + is_latest=True, + ) + self.put_object_version(version_id=obj.version_id, obj=obj) + return + elif versioning.status == SUSPENDED: + if version_id: + latest_obj = self.get_latest() + del self.versions[version_id] + if version_id == latest_obj.version_id: + obj = list(self.versions.values())[0] + obj.is_latest = True + else: + latest_obj = self.get_latest() + latest_obj.is_delete_marker = True + else: + raise Exception("unexpected") + except Exception as e: + logging.error("remove_object(): Exception") + logging.error(e) + raise + class MockMinioBucket: """ @@ -383,63 +447,13 @@ def remove_object(self, object_name, version_id=None): return try: if self.versioning.status == OFF: - if version_id: - if version_id in self.objects[object_name]: - del self.objects[object_name][version_id] - else: + if not version_id: del self.objects[object_name] - return - except Exception as e: - logging.error("remove_object(): Exception") - logging.error(e) - raise e - - try: - if self.versioning.status == ENABLED: - if version_id: - if version_id not in self.objects[object_name]: - # version_id does not exist - # nothing to do - return - else: - latest_obj = self.objects[object_name].get_latest() - del self.objects[object_name][version_id] - if version_id == latest_obj.version_id: - obj = list(self.objects[object_name].values())[0] - obj.is_latest = True - - else: # version_id is False - # - latest_obj = self.objects[object_name].get_latest() - if latest_obj.is_delete_marker: - # nothing to do - return - - self._put_object( - bucket_name=bucket_name, - object_name=object_name, - data=b"", - length=0, - is_delete_marker=True, - ) return - elif self.versioning.status == SUSPENDED: - if version_id: - latest_obj = self.objects[object_name].get_latest() - del self.objects[object_name][version_id] - if version_id == latest_obj.version_id: - obj = list(self.objects[object_name].values())[0] - obj.is_latest = True - - else: - latest_obj = self.objects[object_name].get_latest() - latest_obj.is_delete_marker = True - else: - raise Exception("unexpected") + return self.objects[object_name].remove_object(version_id, self.versioning) except Exception as e: logging.error("remove_object(): Exception") - logging.error(self.buckets) logging.error(e) raise @@ -465,8 +479,8 @@ def get_object(self, object_name, version_id): message=e.message, response=e.response, resource=f"/{self.bucket_name}/{object_name}", - host_id=e._host_id, - request_id=e.request_id, + host_id=None, + request_id=None, code=e.code, bucket_name=self.bucket_name, object_name=object_name, @@ -1203,7 +1217,9 @@ def remove_object(self, bucket_name, object_name, version_id=None): None: The method has no return value but indicates successful removal. """ self._health_check() - return self.buckets[bucket_name].remove_object(object_name, version_id=None) + return self.buckets[bucket_name].remove_object( + object_name, version_id=version_id + ) @pytest.fixture diff --git a/tests/test_minio_mock.py b/tests/test_minio_mock.py index 7b0866e..9fdd8cb 100644 --- a/tests/test_minio_mock.py +++ b/tests/test_minio_mock.py @@ -140,7 +140,7 @@ def test_removing_object_version_with_versionning_enabled(minio_mock): objects = list(client.list_objects(bucket_name, object_name, include_version=True)) assert len(objects) == 1 assert objects[0].version_id == last_version - assert objects[0].is_latest is True + assert objects[0].is_latest == "true" client.fput_object(bucket_name, object_name, file_path) objects = list(client.list_objects(bucket_name, object_name, include_version=True))