[Refactor] Use the new interface of fileio from mmengine (#2468)

* [Refactor] Use new interface of fileio

* update ut

* remove file_client_args

* add * before backend_args

* refine backend_args description

mmcv/image/io.py

@@ -6,10 +6,10 @@
 from typing import Optional, Union
 
 import cv2
+import mmengine.fileio as fileio
 import numpy as np
 from cv2 import (IMREAD_COLOR, IMREAD_GRAYSCALE, IMREAD_IGNORE_ORIENTATION,
                  IMREAD_UNCHANGED)
-from mmengine.fileio import FileClient
 from mmengine.utils import is_filepath, is_str
 
 try:
@@ -145,12 +145,10 @@ def imread(img_or_path: Union[np.ndarray, str, Path],
            flag: str = 'color',
            channel_order: str = 'bgr',
            backend: Optional[str] = None,
-           file_client_args: Optional[dict] = None) -> np.ndarray:
+           *,
+           backend_args: Optional[dict] = None) -> np.ndarray:
     """Read an image.
 
-    Note:
-        In v1.4.1 and later, add `file_client_args` parameters.
-
     Args:
         img_or_path (ndarray or str or Path): Either a numpy array or str or
             pathlib.Path. If it is a numpy array (loaded image), then
@@ -168,9 +166,12 @@ def imread(img_or_path: Union[np.ndarray, str, Path],
             `cv2`, `pillow`, `turbojpeg`, `tifffile`, `None`.
             If backend is None, the global imread_backend specified by
             ``mmcv.use_backend()`` will be used. Default: None.
-        file_client_args (dict | None): Arguments to instantiate a
-            FileClient. See :class:`mmengine.fileio.FileClient` for details.
-            Default: None.
+        backend_args (dict, optional): Instantiates the corresponding file
+            backend. It may contain `backend` key to specify the file
+            backend. If it contains, the file backend corresponding to this
+            value will be used and initialized with the remaining values,
+            otherwise the corresponding file backend will be selected
+            based on the prefix of the file path. Defaults to None.
 
     Returns:
         ndarray: Loaded image array.
@@ -187,22 +188,20 @@ def imread(img_or_path: Union[np.ndarray, str, Path],
         >>> # infer the file backend by the prefix s3
         >>> img = mmcv.imread(s3_img_path)
         >>> # manually set the file backend petrel
-        >>> img = mmcv.imread(s3_img_path, file_client_args={
+        >>> img = mmcv.imread(s3_img_path, backend_args={
         ...     'backend': 'petrel'})
         >>> http_img_path = 'http://path/to/img.jpg'
         >>> img = mmcv.imread(http_img_path)
-        >>> img = mmcv.imread(http_img_path, file_client_args={
+        >>> img = mmcv.imread(http_img_path, backend_args={
         ...     'backend': 'http'})
     """
-
     if isinstance(img_or_path, Path):
         img_or_path = str(img_or_path)
 
     if isinstance(img_or_path, np.ndarray):
         return img_or_path
     elif is_str(img_or_path):
-        file_client = FileClient.infer_client(file_client_args, img_or_path)
-        img_bytes = file_client.get(img_or_path)
+        img_bytes = fileio.get(img_or_path, backend_args=backend_args)
         return imfrombytes(img_bytes, flag, channel_order, backend)
     else:
         raise TypeError('"img" must be a numpy array or a str or '
@@ -272,12 +271,10 @@ def imwrite(img: np.ndarray,
             file_path: str,
             params: Optional[list] = None,
             auto_mkdir: Optional[bool] = None,
-            file_client_args: Optional[dict] = None) -> bool:
+            *,
+            backend_args: Optional[dict] = None) -> bool:
     """Write image to file.
 
-    Note:
-        In v1.4.1 and later, add `file_client_args` parameters.
-
     Warning:
         The parameter `auto_mkdir` will be deprecated in the future and every
         file clients will make directory automatically.
@@ -288,9 +285,12 @@ def imwrite(img: np.ndarray,
         params (None or list): Same as opencv :func:`imwrite` interface.
         auto_mkdir (bool): If the parent folder of `file_path` does not exist,
             whether to create it automatically. It will be deprecated.
-        file_client_args (dict | None): Arguments to instantiate a
-            FileClient. See :class:`mmengine.fileio.FileClient` for details.
-            Default: None.
+        backend_args (dict, optional): Instantiates the corresponding file
+            backend. It may contain `backend` key to specify the file
+            backend. If it contains, the file backend corresponding to this
+            value will be used and initialized with the remaining values,
+            otherwise the corresponding file backend will be selected
+            based on the prefix of the file path. Defaults to None.
 
     Returns:
         bool: Successful or not.
@@ -301,7 +301,7 @@ def imwrite(img: np.ndarray,
         >>> # infer the file backend by the prefix s3
         >>> ret = mmcv.imwrite(img, 's3://bucket/img.jpg')
         >>> # manually set the file backend petrel
-        >>> ret = mmcv.imwrite(img, 's3://bucket/img.jpg', file_client_args={
+        >>> ret = mmcv.imwrite(img, 's3://bucket/img.jpg', backend_args={
         ...     'backend': 'petrel'})
     """
     assert is_filepath(file_path)
@@ -310,11 +310,13 @@ def imwrite(img: np.ndarray,
         warnings.warn(
             'The parameter `auto_mkdir` will be deprecated in the future and '
             'every file clients will make directory automatically.')
-    file_client = FileClient.infer_client(file_client_args, file_path)
+
     img_ext = osp.splitext(file_path)[-1]
     # Encode image according to image suffix.
     # For example, if image path is '/path/your/img.jpg', the encode
     # format is '.jpg'.
     flag, img_buff = cv2.imencode(img_ext, img, params)
-    file_client.put(img_buff.tobytes(), file_path)
+
+    fileio.put(img_buff.tobytes(), file_path, backend_args=backend_args)
+
     return flag

mmcv/transforms/loading.py

@@ -1,7 +1,7 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 from typing import Optional
 
-import mmengine
+import mmengine.fileio as fileio
 import numpy as np
 
 import mmcv
@@ -33,25 +33,28 @@ class LoadImageFromFile(BaseTransform):
             argument for :func:`mmcv.imfrombytes`.
             See :func:`mmcv.imfrombytes` for details.
             Defaults to 'cv2'.
-        file_client_args (dict): Arguments to instantiate a FileClient.
-            See :class:`mmengine.fileio.FileClient` for details.
-            Defaults to ``dict(backend='disk')``.
         ignore_empty (bool): Whether to allow loading empty image or file path
             not existent. Defaults to False.
+        backend_args (dict, optional): Instantiates the corresponding file
+            backend. It may contain `backend` key to specify the file
+            backend. If it contains, the file backend corresponding to this
+            value will be used and initialized with the remaining values,
+            otherwise the corresponding file backend will be selected
+            based on the prefix of the file path. Defaults to None.
     """
 
     def __init__(self,
                  to_float32: bool = False,
                  color_type: str = 'color',
                  imdecode_backend: str = 'cv2',
-                 file_client_args: dict = dict(backend='disk'),
-                 ignore_empty: bool = False) -> None:
+                 ignore_empty: bool = False,
+                 *,
+                 backend_args: Optional[dict] = None) -> None:
         self.ignore_empty = ignore_empty
         self.to_float32 = to_float32
         self.color_type = color_type
         self.imdecode_backend = imdecode_backend
-        self.file_client_args = file_client_args.copy()
-        self.file_client = mmengine.FileClient(**self.file_client_args)
+        self.backend_args = backend_args
 
     def transform(self, results: dict) -> Optional[dict]:
         """Functions to load image.
@@ -66,7 +69,7 @@ def transform(self, results: dict) -> Optional[dict]:
 
         filename = results['img_path']
         try:
-            img_bytes = self.file_client.get(filename)
+            img_bytes = fileio.get(filename, backend_args=self.backend_args)
             img = mmcv.imfrombytes(
                 img_bytes, flag=self.color_type, backend=self.imdecode_backend)
         except Exception as e:
@@ -87,8 +90,13 @@ def __repr__(self):
                     f'ignore_empty={self.ignore_empty}, '
                     f'to_float32={self.to_float32}, '
                     f"color_type='{self.color_type}', "
-                    f"imdecode_backend='{self.imdecode_backend}', "
-                    f'file_client_args={self.file_client_args})')
+                    f"imdecode_backend='{self.imdecode_backend}'")
+
+        if self.backend_args is not None:
+            repr_str += f', backend_args={self.backend_args})'
+        else:
+            repr_str += ')'
+
         return repr_str
 
 
@@ -169,9 +177,12 @@ class LoadAnnotations(BaseTransform):
             argument for :func:`mmcv.imfrombytes`.
             See :func:`mmcv.imfrombytes` for details.
             Defaults to 'cv2'.
-        file_client_args (dict): Arguments to instantiate a FileClient.
-            See :class:`mmengine.fileio.FileClient` for details.
-            Defaults to ``dict(backend='disk')``.
+        backend_args (dict, optional): Instantiates the corresponding file
+            backend. It may contain `backend` key to specify the file
+            backend. If it contains, the file backend corresponding to this
+            value will be used and initialized with the remaining values,
+            otherwise the corresponding file backend will be selected
+            based on the prefix of the file path. Defaults to None.
     """
 
     def __init__(
@@ -181,16 +192,16 @@ def __init__(
         with_seg: bool = False,
         with_keypoints: bool = False,
         imdecode_backend: str = 'cv2',
-        file_client_args: dict = dict(backend='disk')
+        *,
+        backend_args: Optional[dict] = None,
     ) -> None:
         super().__init__()
         self.with_bbox = with_bbox
         self.with_label = with_label
         self.with_seg = with_seg
         self.with_keypoints = with_keypoints
         self.imdecode_backend = imdecode_backend
-        self.file_client_args = file_client_args.copy()
-        self.file_client = mmengine.FileClient(**self.file_client_args)
+        self.backend_args = backend_args
 
     def _load_bboxes(self, results: dict) -> None:
         """Private function to load bounding box annotations.
@@ -235,7 +246,8 @@ def _load_seg_map(self, results: dict) -> None:
             dict: The dict contains loaded semantic segmentation annotations.
         """
 
-        img_bytes = self.file_client.get(results['seg_map_path'])
+        img_bytes = fileio.get(
+            results['seg_map_path'], backend_args=self.backend_args)
         results['gt_seg_map'] = mmcv.imfrombytes(
             img_bytes, flag='unchanged',
             backend=self.imdecode_backend).squeeze()
@@ -284,6 +296,11 @@ def __repr__(self) -> str:
         repr_str += f'with_label={self.with_label}, '
         repr_str += f'with_seg={self.with_seg}, '
         repr_str += f'with_keypoints={self.with_keypoints}, '
-        repr_str += f"imdecode_backend='{self.imdecode_backend}', "
-        repr_str += f'file_client_args={self.file_client_args})'
+        repr_str += f"imdecode_backend='{self.imdecode_backend}'"
+
+        if self.backend_args is not None:
+            repr_str += f', backend_args={self.backend_args})'
+        else:
+            repr_str += ')'
+
         return repr_str

requirements/runtime.txt

@@ -1,5 +1,5 @@
 addict
-mmengine
+mmengine>=0.2.0
 numpy
 packaging
 Pillow

tests/test_image/test_io.py

@@ -98,7 +98,7 @@ def test_imread(self):
             img_cv2_color_bgr_petrel_with_args = mmcv.imread(
                 self.s3_path,
                 backend='cv2',
-                file_client_args={'backend': 'petrel'})
+                backend_args={'backend': 'petrel'})
             mock_method.assert_called()
             assert_array_equal(img_cv2_color_bgr_petrel,
                                img_cv2_color_bgr_petrel_with_args)
@@ -112,7 +112,7 @@ def test_imread(self):
             img_cv2_color_bgr_http_with_args = mmcv.imread(
                 self.http_path,
                 backend='cv2',
-                file_client_args={'backend': 'http'})
+                backend_args={'backend': 'http'})
             mock_method.assert_called()
             assert_array_equal(img_cv2_color_bgr_http,
                                img_cv2_color_bgr_http_with_args)
@@ -357,6 +357,7 @@ def test_imfrombytes(self):
     def test_imwrite(self):
         img = mmcv.imread(self.img_path)
         out_file = osp.join(tempfile.gettempdir(), 'mmcv_test.jpg')
+
         mmcv.imwrite(img, out_file)
         rewrite_img = mmcv.imread(out_file)
         os.remove(out_file)
@@ -366,9 +367,10 @@ def test_imwrite(self):
         with patch.object(
                 PetrelBackend, 'put', return_value=None) as mock_method:
             ret = mmcv.imwrite(img, self.s3_path)
-            ret_with_args = mmcv.imwrite(
-                img, self.s3_path, file_client_args={'backend': 'petrel'})
             assert ret
+
+            ret_with_args = mmcv.imwrite(
+                img, self.s3_path, backend_args={'backend': 'petrel'})
             assert ret_with_args
             mock_method.assert_called()
 

tests/test_transforms/test_transforms_loading.py

@@ -23,7 +23,7 @@ def test_load_img(self):
         assert results['ori_shape'] == (300, 400)
         assert repr(transform) == transform.__class__.__name__ + \
             "(ignore_empty=False, to_float32=False, color_type='color', " + \
-            "imdecode_backend='cv2', file_client_args={'backend': 'disk'})"
+            "imdecode_backend='cv2')"
 
         # to_float32
         transform = LoadImageFromFile(to_float32=True)
@@ -131,5 +131,4 @@ def test_repr(self):
         assert repr(transform) == (
             'LoadAnnotations(with_bbox=True, '
             'with_label=False, with_seg=False, '
-            "with_keypoints=False, imdecode_backend='cv2', "
-            "file_client_args={'backend': 'disk'})")
+            "with_keypoints=False, imdecode_backend='cv2')")