From 79e160611014852c02b8266dfaa516367aa25c3c Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Sun, 4 Dec 2022 16:11:30 +0800 Subject: [PATCH 1/5] [Refactor] Use new interface of fileio --- mmcv/image/io.py | 70 +++++++++++++++++++++++-------- mmcv/transforms/loading.py | 86 ++++++++++++++++++++++++++++++++------ requirements/runtime.txt | 2 +- 3 files changed, 126 insertions(+), 32 deletions(-) diff --git a/mmcv/image/io.py b/mmcv/image/io.py index 8d2c862353..7aeec6ec3c 100644 --- a/mmcv/image/io.py +++ b/mmcv/image/io.py @@ -9,7 +9,7 @@ import numpy as np from cv2 import (IMREAD_COLOR, IMREAD_GRAYSCALE, IMREAD_IGNORE_ORIENTATION, IMREAD_UNCHANGED) -from mmengine.fileio import FileClient +from mmengine.fileio import FileClient, get, put from mmengine.utils import is_filepath, is_str try: @@ -145,12 +145,10 @@ def imread(img_or_path: Union[np.ndarray, str, Path], flag: str = 'color', channel_order: str = 'bgr', backend: Optional[str] = None, - file_client_args: Optional[dict] = None) -> np.ndarray: + file_client_args: Optional[dict] = None, + backend_args: Optional[dict] = None) -> np.ndarray: """Read an image. - Note: - In v1.4.1 and later, add `file_client_args` parameters. - Args: img_or_path (ndarray or str or Path): Either a numpy array or str or pathlib.Path. If it is a numpy array (loaded image), then @@ -170,7 +168,12 @@ def imread(img_or_path: Union[np.ndarray, str, Path], ``mmcv.use_backend()`` will be used. Default: None. file_client_args (dict | None): Arguments to instantiate a FileClient. See :class:`mmengine.fileio.FileClient` for details. - Default: None. + Default: None. It will be deprecated in future. Please use + ``backend_args`` instead. + Deprecated in version 2.0.0rc4. + backend_args (dict, optional): Arguments to instantiate the + preifx of uri corresponding backend. Defaults to None. + New in version 2.0.0rc4. Returns: ndarray: Loaded image array. @@ -187,13 +190,21 @@ def imread(img_or_path: Union[np.ndarray, str, Path], >>> # infer the file backend by the prefix s3 >>> img = mmcv.imread(s3_img_path) >>> # manually set the file backend petrel - >>> img = mmcv.imread(s3_img_path, file_client_args={ + >>> img = mmcv.imread(s3_img_path, backend_args={ ... 'backend': 'petrel'}) >>> http_img_path = 'http://path/to/img.jpg' >>> img = mmcv.imread(http_img_path) - >>> img = mmcv.imread(http_img_path, file_client_args={ + >>> img = mmcv.imread(http_img_path, backend_args={ ... 'backend': 'http'}) """ + if file_client_args is not None: + warnings.warn( + '"file_client_args" will be deprecated in future. ' + 'Please use "backend_args" instead', DeprecationWarning) + if backend_args is not None: + raise ValueError( + '"file_client_args" and "backend_args" cannot be set at the ' + 'same time.') if isinstance(img_or_path, Path): img_or_path = str(img_or_path) @@ -201,8 +212,13 @@ def imread(img_or_path: Union[np.ndarray, str, Path], if isinstance(img_or_path, np.ndarray): return img_or_path elif is_str(img_or_path): - file_client = FileClient.infer_client(file_client_args, img_or_path) - img_bytes = file_client.get(img_or_path) + if file_client_args is not None: + file_client = FileClient.infer_client(file_client_args, + img_or_path) + img_bytes = file_client.get(img_or_path) + else: + img_bytes = get(img_or_path, backend_args=backend_args) + return imfrombytes(img_bytes, flag, channel_order, backend) else: raise TypeError('"img" must be a numpy array or a str or ' @@ -272,12 +288,10 @@ def imwrite(img: np.ndarray, file_path: str, params: Optional[list] = None, auto_mkdir: Optional[bool] = None, - file_client_args: Optional[dict] = None) -> bool: + file_client_args: Optional[dict] = None, + backend_args: Optional[dict] = None) -> bool: """Write image to file. - Note: - In v1.4.1 and later, add `file_client_args` parameters. - Warning: The parameter `auto_mkdir` will be deprecated in the future and every file clients will make directory automatically. @@ -290,7 +304,12 @@ def imwrite(img: np.ndarray, whether to create it automatically. It will be deprecated. file_client_args (dict | None): Arguments to instantiate a FileClient. See :class:`mmengine.fileio.FileClient` for details. - Default: None. + Default: None. It will be deprecated in future. Please use + ``backend_args`` instead. + Deprecated in version 2.0.0rc4. + backend_args (dict, optional): Arguments to instantiate the + preifx of uri corresponding backend. Defaults to None. + New in version 2.0.0rc4. Returns: bool: Successful or not. @@ -301,20 +320,35 @@ def imwrite(img: np.ndarray, >>> # infer the file backend by the prefix s3 >>> ret = mmcv.imwrite(img, 's3://bucket/img.jpg') >>> # manually set the file backend petrel - >>> ret = mmcv.imwrite(img, 's3://bucket/img.jpg', file_client_args={ + >>> ret = mmcv.imwrite(img, 's3://bucket/img.jpg', backend_args={ ... 'backend': 'petrel'}) """ + if file_client_args is not None: + warnings.warn( + '"file_client_args" will be deprecated in future. ' + 'Please use "backend_args" instead', DeprecationWarning) + if backend_args is not None: + raise ValueError( + '"file_client_args" and "backend_args" cannot be set at the ' + 'same time.') + assert is_filepath(file_path) file_path = str(file_path) if auto_mkdir is not None: warnings.warn( 'The parameter `auto_mkdir` will be deprecated in the future and ' 'every file clients will make directory automatically.') - file_client = FileClient.infer_client(file_client_args, file_path) + img_ext = osp.splitext(file_path)[-1] # Encode image according to image suffix. # For example, if image path is '/path/your/img.jpg', the encode # format is '.jpg'. flag, img_buff = cv2.imencode(img_ext, img, params) - file_client.put(img_buff.tobytes(), file_path) + + if file_client_args is not None: + file_client = FileClient.infer_client(file_client_args, file_path) + file_client.put(img_buff.tobytes(), file_path) + else: + put(img_buff.tobytes(), file_path, backend_args=backend_args) + return flag diff --git a/mmcv/transforms/loading.py b/mmcv/transforms/loading.py index f1188837ff..5bc88dd297 100644 --- a/mmcv/transforms/loading.py +++ b/mmcv/transforms/loading.py @@ -1,8 +1,9 @@ # Copyright (c) OpenMMLab. All rights reserved. -from typing import Optional +import warnings +from typing import Optional, Union -import mmengine import numpy as np +from mmengine.fileio import BaseStorageBackend, FileClient, get_file_backend import mmcv from .base import BaseTransform @@ -35,23 +36,50 @@ class LoadImageFromFile(BaseTransform): Defaults to 'cv2'. file_client_args (dict): Arguments to instantiate a FileClient. See :class:`mmengine.fileio.FileClient` for details. - Defaults to ``dict(backend='disk')``. + Defaults to None. It will be deprecated in future. Please use + ``backend_args`` instead. + Deprecated in version 2.0.0rc4. ignore_empty (bool): Whether to allow loading empty image or file path not existent. Defaults to False. + backend_args (dict, optional): Arguments to instantiate the + preifx of uri corresponding backend. Defaults to None. + New in version 2.0.0rc4. """ def __init__(self, to_float32: bool = False, color_type: str = 'color', imdecode_backend: str = 'cv2', - file_client_args: dict = dict(backend='disk'), - ignore_empty: bool = False) -> None: + file_client_args: Optional[dict] = None, + ignore_empty: bool = False, + backend_args: Optional[dict] = None) -> None: self.ignore_empty = ignore_empty self.to_float32 = to_float32 self.color_type = color_type self.imdecode_backend = imdecode_backend + + if file_client_args is not None: + warnings.warn( + '"file_client_args" will be deprecated in future. ' + 'Please use "backend_args" instead', DeprecationWarning) + if backend_args is not None: + raise ValueError( + '"file_client_args" and "backend_args" cannot be set ' + 'at the same time.') + else: + file_client_args = dict(backend='disk') + if backend_args is None: + backend_args = dict(backend='disk') self.file_client_args = file_client_args.copy() - self.file_client = mmengine.FileClient(**self.file_client_args) + self.file_client = FileClient(**self.file_client_args) + self.backend_args = backend_args.copy() + + self.file_backend: Union[FileClient, BaseStorageBackend] + if self.file_client_args is None: + self.file_backend = get_file_backend( + backend_args=self.backend_args) + else: + self.file_backend = self.file_client def transform(self, results: dict) -> Optional[dict]: """Functions to load image. @@ -66,7 +94,7 @@ def transform(self, results: dict) -> Optional[dict]: filename = results['img_path'] try: - img_bytes = self.file_client.get(filename) + img_bytes = self.file_backend.get(filename) img = mmcv.imfrombytes( img_bytes, flag=self.color_type, backend=self.imdecode_backend) except Exception as e: @@ -87,8 +115,13 @@ def __repr__(self): f'ignore_empty={self.ignore_empty}, ' f'to_float32={self.to_float32}, ' f"color_type='{self.color_type}', " - f"imdecode_backend='{self.imdecode_backend}', " - f'file_client_args={self.file_client_args})') + f"imdecode_backend='{self.imdecode_backend}', ") + + if self.file_client_args is not None: + repr_str += f'file_client_args={self.file_client_args})' + else: + repr_str += f'backend_args={self.backend_args})' + return repr_str @@ -181,7 +214,8 @@ def __init__( with_seg: bool = False, with_keypoints: bool = False, imdecode_backend: str = 'cv2', - file_client_args: dict = dict(backend='disk') + file_client_args: Optional[dict] = None, + backend_args: Optional[dict] = None, ) -> None: super().__init__() self.with_bbox = with_bbox @@ -189,8 +223,29 @@ def __init__( self.with_seg = with_seg self.with_keypoints = with_keypoints self.imdecode_backend = imdecode_backend + + if file_client_args is not None: + warnings.warn( + '"file_client_args" will be deprecated in future. ' + 'Please use "backend_args" instead', DeprecationWarning) + if backend_args is not None: + raise ValueError( + '"file_client_args" and "backend_args" cannot be set ' + 'at the same time.') + else: + file_client_args = dict(backend='disk') + if backend_args is None: + backend_args = dict(backend='disk') self.file_client_args = file_client_args.copy() - self.file_client = mmengine.FileClient(**self.file_client_args) + self.file_client = FileClient(**self.file_client_args) + self.backend_args = backend_args.copy() + + self.file_backend: Union[FileClient, BaseStorageBackend] + if self.file_client_args is None: + self.file_backend = get_file_backend( + backend_args=self.backend_args) + else: + self.file_backend = self.file_client def _load_bboxes(self, results: dict) -> None: """Private function to load bounding box annotations. @@ -235,7 +290,7 @@ def _load_seg_map(self, results: dict) -> None: dict: The dict contains loaded semantic segmentation annotations. """ - img_bytes = self.file_client.get(results['seg_map_path']) + img_bytes = self.file_backend.get(results['seg_map_path']) results['gt_seg_map'] = mmcv.imfrombytes( img_bytes, flag='unchanged', backend=self.imdecode_backend).squeeze() @@ -285,5 +340,10 @@ def __repr__(self) -> str: repr_str += f'with_seg={self.with_seg}, ' repr_str += f'with_keypoints={self.with_keypoints}, ' repr_str += f"imdecode_backend='{self.imdecode_backend}', " - repr_str += f'file_client_args={self.file_client_args})' + + if self.file_client_args is not None: + repr_str += f'file_client_args={self.file_client_args})' + else: + repr_str += f'backend_args={self.backend_args})' + return repr_str diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 303c4b015c..167b58c2c2 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -1,5 +1,5 @@ addict -mmengine +mmengine>=0.2.0 numpy packaging Pillow From 1f51ac27208a211651d552e502eddc708995601d Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Sun, 4 Dec 2022 20:57:36 +0800 Subject: [PATCH 2/5] update ut --- tests/test_image/test_io.py | 47 +++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/test_image/test_io.py b/tests/test_image/test_io.py index 4fe1bc8af0..6742924f23 100644 --- a/tests/test_image/test_io.py +++ b/tests/test_image/test_io.py @@ -60,6 +60,15 @@ def test_imread(self): # backend cv2 mmcv.use_backend('cv2') + # file_client_args and backend_args can not be both set + with pytest.raises( + ValueError, + match='"file_client_args" and "backend_args" cannot be set'): + mmcv.imread( + self.img_path, + file_client_args={'backend': 'disk'}, + backend_args={'backend': 'disk'}) + # HardDiskBackend img_cv2_color_bgr = mmcv.imread(self.img_path) assert img_cv2_color_bgr.shape == (300, 400, 3) @@ -103,6 +112,16 @@ def test_imread(self): assert_array_equal(img_cv2_color_bgr_petrel, img_cv2_color_bgr_petrel_with_args) + mock_method.reset_mock() + + img_cv2_color_bgr_petrel_with_args = mmcv.imread( + self.s3_path, + backend='cv2', + backend_args={'backend': 'petrel'}) + mock_method.assert_called() + assert_array_equal(img_cv2_color_bgr_petrel, + img_cv2_color_bgr_petrel_with_args) + # HTTPBackend img_cv2_color_bgr = mmcv.imread(self.img_path) with patch.object( @@ -117,6 +136,16 @@ def test_imread(self): assert_array_equal(img_cv2_color_bgr_http, img_cv2_color_bgr_http_with_args) + mock_method.reset_mock() + + img_cv2_color_bgr_http_with_args = mmcv.imread( + self.http_path, + backend='cv2', + backend_args={'backend': 'http'}) + mock_method.assert_called() + assert_array_equal(img_cv2_color_bgr_http, + img_cv2_color_bgr_http_with_args) + with pytest.raises(FileNotFoundError): mmcv.imread('/not/exists/' + self.img_path) @@ -357,6 +386,17 @@ def test_imfrombytes(self): def test_imwrite(self): img = mmcv.imread(self.img_path) out_file = osp.join(tempfile.gettempdir(), 'mmcv_test.jpg') + + # file_client_args and backend_args can not be both set + with pytest.raises( + ValueError, + match='"file_client_args" and "backend_args" cannot be set'): + mmcv.imwrite( + img, + out_file, + file_client_args={'backend': 'disk'}, + backend_args={'backend': 'disk'}) + mmcv.imwrite(img, out_file) rewrite_img = mmcv.imread(out_file) os.remove(out_file) @@ -372,6 +412,13 @@ def test_imwrite(self): assert ret_with_args mock_method.assert_called() + mock_method.reset_mock() + + ret_with_args = mmcv.imwrite( + img, self.s3_path, backend_args={'backend': 'petrel'}) + assert ret_with_args + mock_method.assert_called() + with pytest.raises(cv2.error): mmcv.imwrite(img, 'error_file.jppg') From 130cb2ec124ca745bd5cd13611fbab0edc54d386 Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Sun, 11 Dec 2022 14:13:05 +0800 Subject: [PATCH 3/5] remove file_client_args --- mmcv/image/io.py | 50 ++--------- mmcv/transforms/loading.py | 88 ++++--------------- tests/test_image/test_io.py | 45 ---------- .../test_transforms_loading.py | 5 +- 4 files changed, 25 insertions(+), 163 deletions(-) diff --git a/mmcv/image/io.py b/mmcv/image/io.py index 7aeec6ec3c..2c50d78f09 100644 --- a/mmcv/image/io.py +++ b/mmcv/image/io.py @@ -6,10 +6,10 @@ from typing import Optional, Union import cv2 +import mmengine.fileio as fileio import numpy as np from cv2 import (IMREAD_COLOR, IMREAD_GRAYSCALE, IMREAD_IGNORE_ORIENTATION, IMREAD_UNCHANGED) -from mmengine.fileio import FileClient, get, put from mmengine.utils import is_filepath, is_str try: @@ -145,7 +145,7 @@ def imread(img_or_path: Union[np.ndarray, str, Path], flag: str = 'color', channel_order: str = 'bgr', backend: Optional[str] = None, - file_client_args: Optional[dict] = None, + *, backend_args: Optional[dict] = None) -> np.ndarray: """Read an image. @@ -166,14 +166,8 @@ def imread(img_or_path: Union[np.ndarray, str, Path], `cv2`, `pillow`, `turbojpeg`, `tifffile`, `None`. If backend is None, the global imread_backend specified by ``mmcv.use_backend()`` will be used. Default: None. - file_client_args (dict | None): Arguments to instantiate a - FileClient. See :class:`mmengine.fileio.FileClient` for details. - Default: None. It will be deprecated in future. Please use - ``backend_args`` instead. - Deprecated in version 2.0.0rc4. backend_args (dict, optional): Arguments to instantiate the preifx of uri corresponding backend. Defaults to None. - New in version 2.0.0rc4. Returns: ndarray: Loaded image array. @@ -197,28 +191,13 @@ def imread(img_or_path: Union[np.ndarray, str, Path], >>> img = mmcv.imread(http_img_path, backend_args={ ... 'backend': 'http'}) """ - if file_client_args is not None: - warnings.warn( - '"file_client_args" will be deprecated in future. ' - 'Please use "backend_args" instead', DeprecationWarning) - if backend_args is not None: - raise ValueError( - '"file_client_args" and "backend_args" cannot be set at the ' - 'same time.') - if isinstance(img_or_path, Path): img_or_path = str(img_or_path) if isinstance(img_or_path, np.ndarray): return img_or_path elif is_str(img_or_path): - if file_client_args is not None: - file_client = FileClient.infer_client(file_client_args, - img_or_path) - img_bytes = file_client.get(img_or_path) - else: - img_bytes = get(img_or_path, backend_args=backend_args) - + img_bytes = fileio.get(img_or_path, backend_args=backend_args) return imfrombytes(img_bytes, flag, channel_order, backend) else: raise TypeError('"img" must be a numpy array or a str or ' @@ -288,7 +267,7 @@ def imwrite(img: np.ndarray, file_path: str, params: Optional[list] = None, auto_mkdir: Optional[bool] = None, - file_client_args: Optional[dict] = None, + *, backend_args: Optional[dict] = None) -> bool: """Write image to file. @@ -302,14 +281,8 @@ def imwrite(img: np.ndarray, params (None or list): Same as opencv :func:`imwrite` interface. auto_mkdir (bool): If the parent folder of `file_path` does not exist, whether to create it automatically. It will be deprecated. - file_client_args (dict | None): Arguments to instantiate a - FileClient. See :class:`mmengine.fileio.FileClient` for details. - Default: None. It will be deprecated in future. Please use - ``backend_args`` instead. - Deprecated in version 2.0.0rc4. backend_args (dict, optional): Arguments to instantiate the preifx of uri corresponding backend. Defaults to None. - New in version 2.0.0rc4. Returns: bool: Successful or not. @@ -323,15 +296,6 @@ def imwrite(img: np.ndarray, >>> ret = mmcv.imwrite(img, 's3://bucket/img.jpg', backend_args={ ... 'backend': 'petrel'}) """ - if file_client_args is not None: - warnings.warn( - '"file_client_args" will be deprecated in future. ' - 'Please use "backend_args" instead', DeprecationWarning) - if backend_args is not None: - raise ValueError( - '"file_client_args" and "backend_args" cannot be set at the ' - 'same time.') - assert is_filepath(file_path) file_path = str(file_path) if auto_mkdir is not None: @@ -345,10 +309,6 @@ def imwrite(img: np.ndarray, # format is '.jpg'. flag, img_buff = cv2.imencode(img_ext, img, params) - if file_client_args is not None: - file_client = FileClient.infer_client(file_client_args, file_path) - file_client.put(img_buff.tobytes(), file_path) - else: - put(img_buff.tobytes(), file_path, backend_args=backend_args) + fileio.put(img_buff.tobytes(), file_path, backend_args=backend_args) return flag diff --git a/mmcv/transforms/loading.py b/mmcv/transforms/loading.py index 5bc88dd297..4e11f845f4 100644 --- a/mmcv/transforms/loading.py +++ b/mmcv/transforms/loading.py @@ -1,9 +1,8 @@ # Copyright (c) OpenMMLab. All rights reserved. -import warnings -from typing import Optional, Union +from typing import Optional +import mmengine.fileio as fileio import numpy as np -from mmengine.fileio import BaseStorageBackend, FileClient, get_file_backend import mmcv from .base import BaseTransform @@ -34,52 +33,24 @@ class LoadImageFromFile(BaseTransform): argument for :func:`mmcv.imfrombytes`. See :func:`mmcv.imfrombytes` for details. Defaults to 'cv2'. - file_client_args (dict): Arguments to instantiate a FileClient. - See :class:`mmengine.fileio.FileClient` for details. - Defaults to None. It will be deprecated in future. Please use - ``backend_args`` instead. - Deprecated in version 2.0.0rc4. ignore_empty (bool): Whether to allow loading empty image or file path not existent. Defaults to False. backend_args (dict, optional): Arguments to instantiate the preifx of uri corresponding backend. Defaults to None. - New in version 2.0.0rc4. """ def __init__(self, to_float32: bool = False, color_type: str = 'color', imdecode_backend: str = 'cv2', - file_client_args: Optional[dict] = None, ignore_empty: bool = False, + *, backend_args: Optional[dict] = None) -> None: self.ignore_empty = ignore_empty self.to_float32 = to_float32 self.color_type = color_type self.imdecode_backend = imdecode_backend - - if file_client_args is not None: - warnings.warn( - '"file_client_args" will be deprecated in future. ' - 'Please use "backend_args" instead', DeprecationWarning) - if backend_args is not None: - raise ValueError( - '"file_client_args" and "backend_args" cannot be set ' - 'at the same time.') - else: - file_client_args = dict(backend='disk') - if backend_args is None: - backend_args = dict(backend='disk') - self.file_client_args = file_client_args.copy() - self.file_client = FileClient(**self.file_client_args) - self.backend_args = backend_args.copy() - - self.file_backend: Union[FileClient, BaseStorageBackend] - if self.file_client_args is None: - self.file_backend = get_file_backend( - backend_args=self.backend_args) - else: - self.file_backend = self.file_client + self.backend_args = backend_args def transform(self, results: dict) -> Optional[dict]: """Functions to load image. @@ -94,7 +65,7 @@ def transform(self, results: dict) -> Optional[dict]: filename = results['img_path'] try: - img_bytes = self.file_backend.get(filename) + img_bytes = fileio.get(filename, backend_args=self.backend_args) img = mmcv.imfrombytes( img_bytes, flag=self.color_type, backend=self.imdecode_backend) except Exception as e: @@ -115,12 +86,12 @@ def __repr__(self): f'ignore_empty={self.ignore_empty}, ' f'to_float32={self.to_float32}, ' f"color_type='{self.color_type}', " - f"imdecode_backend='{self.imdecode_backend}', ") + f"imdecode_backend='{self.imdecode_backend}'") - if self.file_client_args is not None: - repr_str += f'file_client_args={self.file_client_args})' + if self.backend_args is not None: + repr_str += f', backend_args={self.backend_args})' else: - repr_str += f'backend_args={self.backend_args})' + repr_str += ')' return repr_str @@ -202,9 +173,8 @@ class LoadAnnotations(BaseTransform): argument for :func:`mmcv.imfrombytes`. See :func:`mmcv.imfrombytes` for details. Defaults to 'cv2'. - file_client_args (dict): Arguments to instantiate a FileClient. - See :class:`mmengine.fileio.FileClient` for details. - Defaults to ``dict(backend='disk')``. + backend_args (dict, optional): Arguments to instantiate the + preifx of uri corresponding backend. Defaults to None. """ def __init__( @@ -214,7 +184,6 @@ def __init__( with_seg: bool = False, with_keypoints: bool = False, imdecode_backend: str = 'cv2', - file_client_args: Optional[dict] = None, backend_args: Optional[dict] = None, ) -> None: super().__init__() @@ -223,29 +192,7 @@ def __init__( self.with_seg = with_seg self.with_keypoints = with_keypoints self.imdecode_backend = imdecode_backend - - if file_client_args is not None: - warnings.warn( - '"file_client_args" will be deprecated in future. ' - 'Please use "backend_args" instead', DeprecationWarning) - if backend_args is not None: - raise ValueError( - '"file_client_args" and "backend_args" cannot be set ' - 'at the same time.') - else: - file_client_args = dict(backend='disk') - if backend_args is None: - backend_args = dict(backend='disk') - self.file_client_args = file_client_args.copy() - self.file_client = FileClient(**self.file_client_args) - self.backend_args = backend_args.copy() - - self.file_backend: Union[FileClient, BaseStorageBackend] - if self.file_client_args is None: - self.file_backend = get_file_backend( - backend_args=self.backend_args) - else: - self.file_backend = self.file_client + self.backend_args = backend_args def _load_bboxes(self, results: dict) -> None: """Private function to load bounding box annotations. @@ -290,7 +237,8 @@ def _load_seg_map(self, results: dict) -> None: dict: The dict contains loaded semantic segmentation annotations. """ - img_bytes = self.file_backend.get(results['seg_map_path']) + img_bytes = fileio.get( + results['seg_map_path'], backend_args=self.backend_args) results['gt_seg_map'] = mmcv.imfrombytes( img_bytes, flag='unchanged', backend=self.imdecode_backend).squeeze() @@ -339,11 +287,11 @@ def __repr__(self) -> str: repr_str += f'with_label={self.with_label}, ' repr_str += f'with_seg={self.with_seg}, ' repr_str += f'with_keypoints={self.with_keypoints}, ' - repr_str += f"imdecode_backend='{self.imdecode_backend}', " + repr_str += f"imdecode_backend='{self.imdecode_backend}'" - if self.file_client_args is not None: - repr_str += f'file_client_args={self.file_client_args})' + if self.backend_args is not None: + repr_str += f', backend_args={self.backend_args})' else: - repr_str += f'backend_args={self.backend_args})' + repr_str += ')' return repr_str diff --git a/tests/test_image/test_io.py b/tests/test_image/test_io.py index 6742924f23..fce3f1ac0c 100644 --- a/tests/test_image/test_io.py +++ b/tests/test_image/test_io.py @@ -60,15 +60,6 @@ def test_imread(self): # backend cv2 mmcv.use_backend('cv2') - # file_client_args and backend_args can not be both set - with pytest.raises( - ValueError, - match='"file_client_args" and "backend_args" cannot be set'): - mmcv.imread( - self.img_path, - file_client_args={'backend': 'disk'}, - backend_args={'backend': 'disk'}) - # HardDiskBackend img_cv2_color_bgr = mmcv.imread(self.img_path) assert img_cv2_color_bgr.shape == (300, 400, 3) @@ -104,16 +95,6 @@ def test_imread(self): PetrelBackend, 'get', return_value=img_cv2_color_bgr) as mock_method: img_cv2_color_bgr_petrel = mmcv.imread(self.s3_path, backend='cv2') - img_cv2_color_bgr_petrel_with_args = mmcv.imread( - self.s3_path, - backend='cv2', - file_client_args={'backend': 'petrel'}) - mock_method.assert_called() - assert_array_equal(img_cv2_color_bgr_petrel, - img_cv2_color_bgr_petrel_with_args) - - mock_method.reset_mock() - img_cv2_color_bgr_petrel_with_args = mmcv.imread( self.s3_path, backend='cv2', @@ -128,16 +109,6 @@ def test_imread(self): HTTPBackend, 'get', return_value=img_cv2_color_bgr) as mock_method: img_cv2_color_bgr_http = mmcv.imread(self.http_path, backend='cv2') - img_cv2_color_bgr_http_with_args = mmcv.imread( - self.http_path, - backend='cv2', - file_client_args={'backend': 'http'}) - mock_method.assert_called() - assert_array_equal(img_cv2_color_bgr_http, - img_cv2_color_bgr_http_with_args) - - mock_method.reset_mock() - img_cv2_color_bgr_http_with_args = mmcv.imread( self.http_path, backend='cv2', @@ -387,16 +358,6 @@ def test_imwrite(self): img = mmcv.imread(self.img_path) out_file = osp.join(tempfile.gettempdir(), 'mmcv_test.jpg') - # file_client_args and backend_args can not be both set - with pytest.raises( - ValueError, - match='"file_client_args" and "backend_args" cannot be set'): - mmcv.imwrite( - img, - out_file, - file_client_args={'backend': 'disk'}, - backend_args={'backend': 'disk'}) - mmcv.imwrite(img, out_file) rewrite_img = mmcv.imread(out_file) os.remove(out_file) @@ -406,13 +367,7 @@ def test_imwrite(self): with patch.object( PetrelBackend, 'put', return_value=None) as mock_method: ret = mmcv.imwrite(img, self.s3_path) - ret_with_args = mmcv.imwrite( - img, self.s3_path, file_client_args={'backend': 'petrel'}) assert ret - assert ret_with_args - mock_method.assert_called() - - mock_method.reset_mock() ret_with_args = mmcv.imwrite( img, self.s3_path, backend_args={'backend': 'petrel'}) diff --git a/tests/test_transforms/test_transforms_loading.py b/tests/test_transforms/test_transforms_loading.py index 65fcc565aa..fcbafce061 100644 --- a/tests/test_transforms/test_transforms_loading.py +++ b/tests/test_transforms/test_transforms_loading.py @@ -23,7 +23,7 @@ def test_load_img(self): assert results['ori_shape'] == (300, 400) assert repr(transform) == transform.__class__.__name__ + \ "(ignore_empty=False, to_float32=False, color_type='color', " + \ - "imdecode_backend='cv2', file_client_args={'backend': 'disk'})" + "imdecode_backend='cv2')" # to_float32 transform = LoadImageFromFile(to_float32=True) @@ -131,5 +131,4 @@ def test_repr(self): assert repr(transform) == ( 'LoadAnnotations(with_bbox=True, ' 'with_label=False, with_seg=False, ' - "with_keypoints=False, imdecode_backend='cv2', " - "file_client_args={'backend': 'disk'})") + "with_keypoints=False, imdecode_backend='cv2')") From 1cc5ed44ed515ab8fe4d5b1029c5350d39909f7e Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Mon, 26 Dec 2022 20:13:25 +0800 Subject: [PATCH 4/5] add * before backend_args --- mmcv/transforms/loading.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mmcv/transforms/loading.py b/mmcv/transforms/loading.py index 4e11f845f4..38854202a4 100644 --- a/mmcv/transforms/loading.py +++ b/mmcv/transforms/loading.py @@ -184,6 +184,7 @@ def __init__( with_seg: bool = False, with_keypoints: bool = False, imdecode_backend: str = 'cv2', + *, backend_args: Optional[dict] = None, ) -> None: super().__init__() From 69bf88557895717f8dd55a8321b3ca7a4b2d1ab2 Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Tue, 27 Dec 2022 20:51:46 +0800 Subject: [PATCH 5/5] refine backend_args description --- mmcv/image/io.py | 16 ++++++++++++---- mmcv/transforms/loading.py | 16 ++++++++++++---- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/mmcv/image/io.py b/mmcv/image/io.py index 2c50d78f09..fa7b398534 100644 --- a/mmcv/image/io.py +++ b/mmcv/image/io.py @@ -166,8 +166,12 @@ def imread(img_or_path: Union[np.ndarray, str, Path], `cv2`, `pillow`, `turbojpeg`, `tifffile`, `None`. If backend is None, the global imread_backend specified by ``mmcv.use_backend()`` will be used. Default: None. - backend_args (dict, optional): Arguments to instantiate the - preifx of uri corresponding backend. Defaults to None. + backend_args (dict, optional): Instantiates the corresponding file + backend. It may contain `backend` key to specify the file + backend. If it contains, the file backend corresponding to this + value will be used and initialized with the remaining values, + otherwise the corresponding file backend will be selected + based on the prefix of the file path. Defaults to None. Returns: ndarray: Loaded image array. @@ -281,8 +285,12 @@ def imwrite(img: np.ndarray, params (None or list): Same as opencv :func:`imwrite` interface. auto_mkdir (bool): If the parent folder of `file_path` does not exist, whether to create it automatically. It will be deprecated. - backend_args (dict, optional): Arguments to instantiate the - preifx of uri corresponding backend. Defaults to None. + backend_args (dict, optional): Instantiates the corresponding file + backend. It may contain `backend` key to specify the file + backend. If it contains, the file backend corresponding to this + value will be used and initialized with the remaining values, + otherwise the corresponding file backend will be selected + based on the prefix of the file path. Defaults to None. Returns: bool: Successful or not. diff --git a/mmcv/transforms/loading.py b/mmcv/transforms/loading.py index 38854202a4..135d0de04d 100644 --- a/mmcv/transforms/loading.py +++ b/mmcv/transforms/loading.py @@ -35,8 +35,12 @@ class LoadImageFromFile(BaseTransform): Defaults to 'cv2'. ignore_empty (bool): Whether to allow loading empty image or file path not existent. Defaults to False. - backend_args (dict, optional): Arguments to instantiate the - preifx of uri corresponding backend. Defaults to None. + backend_args (dict, optional): Instantiates the corresponding file + backend. It may contain `backend` key to specify the file + backend. If it contains, the file backend corresponding to this + value will be used and initialized with the remaining values, + otherwise the corresponding file backend will be selected + based on the prefix of the file path. Defaults to None. """ def __init__(self, @@ -173,8 +177,12 @@ class LoadAnnotations(BaseTransform): argument for :func:`mmcv.imfrombytes`. See :func:`mmcv.imfrombytes` for details. Defaults to 'cv2'. - backend_args (dict, optional): Arguments to instantiate the - preifx of uri corresponding backend. Defaults to None. + backend_args (dict, optional): Instantiates the corresponding file + backend. It may contain `backend` key to specify the file + backend. If it contains, the file backend corresponding to this + value will be used and initialized with the remaining values, + otherwise the corresponding file backend will be selected + based on the prefix of the file path. Defaults to None. """ def __init__(