update template (#17)

* update template

* Apply suggestions from code review

* Apply suggestions from code review

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* black preview

* fix pylint issues

---------

Co-authored-by: skshetry <skshetry@users.noreply.github.com>
Co-authored-by: skshetry <18718008+skshetry@users.noreply.github.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Saugat Pachhai (सौगात) <suagatchhetri@outlook.com>

.cruft.json

@@ -1,6 +1,6 @@
 {
   "template": "https://github.com/iterative/py-template",
-  "commit": "5fb8af99d07678287dfc10ad1394452b57d23970",
+  "commit": "c4e24f909659b6ce9c34a1da631290f0c70ff2f2",
   "checkout": null,
   "context": {
     "cookiecutter": {

.pre-commit-config.yaml

@@ -2,11 +2,11 @@ default_language_version:
   python: python3
 repos:
   - repo: https://github.com/psf/black
-    rev: 22.8.0
+    rev: 23.1.0
     hooks:
       - id: black
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -29,21 +29,21 @@ repos:
       - id: codespell
         additional_dependencies: ["tomli"]
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.38.0
+    rev: v3.3.1
     hooks:
       - id: pyupgrade
+        args: [--py38-plus]
   - repo: https://github.com/PyCQA/isort
-    rev: 5.10.1
+    rev: 5.12.0
     hooks:
       - id: isort
   - repo: https://github.com/pycqa/flake8
-    rev: 5.0.4
+    rev: 6.0.0
     hooks:
       - id: flake8
         additional_dependencies:
-          - flake8-broken-line==0.5.0
-          - flake8-bugbear==22.9.11
-          - flake8-comprehensions==3.10.0
+          - flake8-bugbear==23.1.20
+          - flake8-comprehensions==3.10.1
           - flake8-debugger==4.1.2
           - flake8-string-format==0.3.0
   - repo: https://github.com/pycqa/bandit

pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [tool.setuptools_scm]
 
 [tool.black]
-line-length = 79
+line-length = 88
 include = '\.pyi?$'
 exclude = '''
 /(
@@ -25,7 +25,7 @@ exclude = '''
 [tool.isort]
 profile = "black"
 known_first_party = ["morefs"]
-line_length = 79
+line_length = 88
 
 [tool.pytest.ini_options]
 addopts = "-ra"
@@ -64,6 +64,9 @@ warn_unreachable = true
 ignore_missing_imports = true
 files = ["src", "tests"]
 
+[tool.pylint.format]
+max-line-length = 88
+
 [tool.pylint.message_control]
 disable = [
     "format", "refactoring", "spelling", "design", "invalid-name",

setup.cfg

@@ -39,7 +39,7 @@ all =
     %(memfs)s
     %(asynclocalfs)s
 tests =
-    pytest==7.1.2
+    pytest==7.2.0
     pytest-sugar==0.9.5
     pytest-cov==3.0.0
     pytest-mock==3.8.2
@@ -67,12 +67,16 @@ where=src
 
 [flake8]
 ignore=
-    E203, # Whitespace before ':'
-    E266, # Too many leading '#' for block comment
-    W503, # Line break occurred before a binary operator
-    P1,  # unindexed parameters in the str.format, see:
+    # Whitespace before ':'
+    E203
+    # Too many leading '#' for block comment
+    E266
+    # Line break occurred before a binary operator
+    W503
+    # unindexed parameters in the str.format, see:
     # https://pypi.org/project/flake8-string-format/
-max_line_length = 79
+    P1
+max_line_length = 88
 max-complexity = 15
 select = B,C,E,F,W,T4,B902,T,P
 show_source = true

src/morefs/asyn_local.py

@@ -59,9 +59,7 @@ class AsyncLocalFileSystem(AsyncFileSystem, LocalFileSystem):
     _write_text = wrap(LocalFileSystem.write_text)
     sign = LocalFileSystem.sign
 
-    async def _get_file(
-        self, src, dst, **kwargs
-    ):  # pylint: disable=arguments-renamed
+    async def _get_file(self, src, dst, **kwargs):  # pylint: disable=arguments-renamed
         if not iscoroutinefunction(getattr(dst, "write", None)):
             src = self._strip_protocol(src)
             return await self._get_file_async(src, dst)

src/morefs/dict.py

@@ -18,9 +18,7 @@ def __init__(self, paths: Iterable[str] = ()) -> None:
     def new_child(self, paths: Iterable[str]) -> None:
         self.set(paths, type(self)(paths=paths))
 
-    def set(
-        self, paths: Iterable[str], value: Any, overwrite: bool = False
-    ) -> None:
+    def set(self, paths: Iterable[str], value: Any, overwrite: bool = False) -> None:
         if not paths:
             raise ValueError("no path supplied")
 
@@ -31,9 +29,7 @@ def set(
             raise ValueError("cannot overwrite - item exists")
         child[key] = value
 
-    def get(  # type: ignore[override]
-        self, paths: Iterable[str]
-    ) -> "ContainerOrFile":
+    def get(self, paths: Iterable[str]) -> "ContainerOrFile":  # type: ignore[override]
         child = self
         for path in paths:
             child = child[path]
@@ -67,18 +63,14 @@ def _strip_protocol(cls, path: str) -> str:
         path = path.lstrip("/").rstrip("/")
         return "/" + path if path else cls.root_marker
 
-    def __init__(self, store: Store = None) -> None:
+    def __init__(self, store: Optional[Store] = None) -> None:
         super().__init__()
         if store is None:
             store = Store()
         self.store = store
 
     def _info(
-        self,
-        path: str,
-        item: ContainerOrFile,
-        file: bool = False,
-        **kwargs: Any
+        self, path: str, item: ContainerOrFile, file: bool = False, **kwargs: Any
     ) -> Dict[str, Any]:
         if isinstance(item, dict):
             return {"name": path, "size": 0, "type": "directory"}
@@ -135,8 +127,7 @@ def ls(self, path: str, detail: bool = False, **kwargs: Any):
         if not detail:
             return [self.join_paths((*paths, key)) for key, _ in entries]
         return [
-            self._info(self.join_paths((*paths, key)), value)
-            for key, value in entries
+            self._info(self.join_paths((*paths, key)), value) for key, value in entries
         ]
 
     def _rm(self, path: str) -> None:
@@ -260,7 +251,7 @@ def rm(
         self,
         path: Union[str, List[str]],
         recursive: bool = False,
-        maxdepth: int = None,
+        maxdepth: Optional[int] = None,
     ) -> None:
         if isinstance(path, str):
             paths = [path]

src/morefs/memory.py

@@ -28,9 +28,11 @@ def _info(path, filelike=None, **kwargs):
         if filelike:
             return {
                 "name": path,
-                "size": filelike.size
-                if hasattr(filelike, "size")
-                else filelike.getbuffer().nbytes,
+                "size": (
+                    filelike.size
+                    if hasattr(filelike, "size")
+                    else filelike.getbuffer().nbytes
+                ),
                 "type": "file",
                 "created": getattr(filelike, "created", None),
             }
@@ -54,9 +56,7 @@ def node_factory(path_conv, parts, children, filelike=None):
         except KeyError as exc:
             if path in ("", "/"):
                 return []
-            raise FileNotFoundError(
-                errno.ENOENT, "No such file", path
-            ) from exc
+            raise FileNotFoundError(errno.ENOENT, "No such file", path) from exc
 
         return out
 
@@ -87,9 +87,7 @@ def rm(self, path, recursive=False, maxdepth=None):
         for p in paths:
             self.store.pop(p, None)
 
-    def _open(  # pylint: disable=arguments-differ
-        self, path, mode="rb", **kwargs
-    ):
+    def _open(self, path, mode="rb", **kwargs):  # pylint: disable=arguments-differ
         path = self._strip_protocol(path)
         try:
             info = self.info(path)

src/morefs/overlay.py

@@ -9,13 +9,12 @@
 class OverlayFileSystem(fsspec.AbstractFileSystem):
     cachable = False
 
-    def __init__(self, *fses, **kwargs):
+    def __init__(self, *fses: fsspec.AbstractFileSystem, **kwargs):
         storage_options = {
-            key: value
-            for key, value in kwargs.items()
-            if key.startswith("fs_")
+            key: value for key, value in kwargs.items() if key.startswith("fs_")
         }
         self.fses: List[fsspec.AbstractFileSystem] = list(fses)
+        self.fses.extend(kwargs.pop("filesystems", []))
         for proto, options in kwargs.items():
             if proto.startswith("fs_"):
                 continue
@@ -79,21 +78,17 @@ def inner(self, path, *args, **kwargs):
     def _raise_readonly(path, *args, **kwargs):
         raise OSError(errno.EROFS, os.strerror(errno.EROFS), path)
 
-    info = _iterate_fs_with("info")
-    created = _iterate_fs_with("created")
-    modified = _iterate_fs_with("modified")
+    info = _iterate_fs_with("info")  # pylint: disable=no-value-for-parameter
+    created = _iterate_fs_with("created")  # pylint: disable=no-value-for-parameter
+    modified = _iterate_fs_with("modified")  # pylint: disable=no-value-for-parameter
 
     def mkdir(self, path, create_parents=True, **kwargs):
         # if create_parents is False:
         if self.exists(path):
-            raise FileExistsError(
-                errno.EEXIST, os.strerror(errno.EEXIST), path
-            )
+            raise FileExistsError(errno.EEXIST, os.strerror(errno.EEXIST), path)
         parent = self._parent(path)
         if not create_parents and not self.isdir(parent):
-            raise FileNotFoundError(
-                errno.ENOENT, os.strerror(errno.ENOENT), path
-            )
+            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path)
         self.upper_fs.mkdir(path, create_parents=True, **kwargs)
 
     def makedirs(self, path, exist_ok=False):
@@ -113,18 +108,14 @@ def cp_file(self, path1, path2, **kwargs):
                 break
 
         if not src_fs:
-            raise FileNotFoundError(
-                errno.ENOENT, os.strerror(errno.ENOENT), path1
-            )
+            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path1)
         if src_fs == self.upper_fs:
             return src_fs.cp_file(path1, path2)
 
         with src_fs.open(path1) as src, self.upper_fs.open(path2, "wb") as dst:
             shutil.copyfileobj(src, dst)
 
-    def _open(
-        self, path, mode="rb", **kwargs
-    ):  # pylint: disable=arguments-differ
+    def _open(self, path, mode="rb", **kwargs):  # pylint: disable=arguments-differ
         if "rb" in mode:
             for fs in self.fses:
                 try:
@@ -165,3 +156,9 @@ def _open(
 
     def sign(self, path, expiration=100, **kwargs):
         return self.upper_fs.sign(path, expiration, **kwargs)
+
+    if hasattr(fsspec.AbstractFileSystem, "fsid"):
+
+        @property
+        def fsid(self):
+            return "overlay_" + "+".join(fs.fsid for fs in self.fses)

tests/test_asyn_local.py

@@ -29,23 +29,17 @@ async def test_ls(tmp_path, localfs, fs):
     assert set(await fs._ls(tmp_path, detail=False)) == {
         localfs._strip_protocol(tmp_path / f) for f in ["foo", "bar", "dir"]
     }
-    assert await fs._ls(tmp_path, detail=False) == localfs.ls(
-        tmp_path, detail=False
-    )
+    assert await fs._ls(tmp_path, detail=False) == localfs.ls(tmp_path, detail=False)
 
     assert await fs._info(tmp_path / "foo") == localfs.info(tmp_path / "foo")
     assert await fs._info(tmp_path / "dir") == localfs.info(tmp_path / "dir")
 
-    assert await fs._ls(tmp_path, detail=True) == localfs.ls(
-        tmp_path, detail=True
-    )
+    assert await fs._ls(tmp_path, detail=True) == localfs.ls(tmp_path, detail=True)
 
     assert await fs._find(tmp_path, detail=False) == localfs.find(
         tmp_path, detail=False
     )
-    assert await fs._find(tmp_path, detail=True) == localfs.find(
-        tmp_path, detail=True
-    )
+    assert await fs._find(tmp_path, detail=True) == localfs.find(tmp_path, detail=True)
 
     assert await fs._isfile(tmp_path / "foo")
     assert await fs._isdir(tmp_path / "dir")
@@ -72,12 +66,8 @@ def test_sync_methods(tmp_path, localfs, fs):
     assert fs.info(tmp_path / "dir") == localfs.info(tmp_path / "dir")
 
     assert fs.ls(tmp_path, detail=True) == localfs.ls(tmp_path, detail=True)
-    assert fs.find(tmp_path, detail=False) == localfs.find(
-        tmp_path, detail=False
-    )
-    assert fs.find(tmp_path, detail=True) == localfs.find(
-        tmp_path, detail=True
-    )
+    assert fs.find(tmp_path, detail=False) == localfs.find(tmp_path, detail=False)
+    assert fs.find(tmp_path, detail=True) == localfs.find(tmp_path, detail=True)
 
     assert fs.isfile(tmp_path / "foo")
     assert fs.isdir(tmp_path / "dir")

tests/test_morefs.py

@@ -6,9 +6,7 @@
 from morefs.memory import MemFS
 
 
-@pytest.mark.parametrize(
-    "proto, fs_cls", [("dictfs", DictFS), ("memfs", MemFS)]
-)
+@pytest.mark.parametrize("proto, fs_cls", [("dictfs", DictFS), ("memfs", MemFS)])
 def test_fsspec(proto, fs_cls):
     fs = fsspec.filesystem(proto)
     assert isinstance(fs, fs_cls)