[STYLE] use ruff in lieu of black/flake8/isort + add pre-commit (#219)

* appease mypy

* add ruff configs

* check and format with ruff

* add ruff and remove black/isort/flake8

* add pre-commit and instructions

* ruff ignore scripts/ + fix with ruff

* install all deps for mypy + add "all" extra

* do not check mypy in nightly pytorch

.github/workflows/ci.yml

@@ -43,8 +43,6 @@ jobs:
           python -m pip install --upgrade pip setuptools wheel
           python -m pip install --pre torch torchvision --extra-index-url https://download.pytorch.org/whl/nightly/cpu openslide-python tiffslide
           python -m pip install --editable .[dev]
-      - name: Check types
-        run: python -m mypy --install-types --non-interactive wsinfer/
       - name: Run tests
         run: python -m pytest --verbose tests/
 
@@ -71,7 +69,6 @@ jobs:
           test -f results/model-outputs-csv/JP2K-33003-1.csv
           test $(wc -l < results/model-outputs-csv/JP2K-33003-1.csv) -eq 675
 
-  # This is run on multiple operating systems.
   test-package:
     strategy:
         matrix:
@@ -121,7 +118,7 @@ jobs:
           Test-Path -Path results/model-outputs-csv/JP2K-33003-1.csv -PathType Leaf
           # test $(python -c "print(sum(1 for _ in open('results/model-outputs/JP2K-33003-1.csv')))") -eq 675
 
-  style-and-types:
+  mypy:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -135,13 +132,9 @@ jobs:
           sudo apt install -y libopenslide0
           python -m pip install --upgrade pip setuptools wheel
           python -m pip install torch torchvision --extra-index-url https://download.pytorch.org/whl/cpu openslide-python tiffslide
-          python -m pip install .[dev]
-      - name: Check style (flake8)
-        run: python -m flake8 wsinfer/
-      - name: Check style (black)
-        run: python -m black --check wsinfer/
+          python -m pip install -e .[all]
       - name: Check types
-        run: python -m mypy --install-types --non-interactive wsinfer/
+        run: python -m mypy --install-types --non-interactive wsinfer/ tests/
 
   docs:
     runs-on: ubuntu-latest

.github/workflows/ruff.yml

@@ -0,0 +1,12 @@
+name: Ruff
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+jobs:
+  ruff:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: chartboost/ruff-action@v1

.pre-commit-config.yaml

@@ -0,0 +1,14 @@
+repos:
+  - repo: 'https://github.com/pre-commit/pre-commit-hooks'
+    rev: v2.3.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: 'https://github.com/astral-sh/ruff-pre-commit'
+    rev: v0.2.2
+    hooks:
+      - id: ruff
+        args:
+          - '--fix'
+      - id: ruff-format

README.md

@@ -86,8 +86,11 @@ Clone this GitHub repository and install the package (in editable mode with the
 git clone https://github.com/SBU-BMI/wsinfer.git
 cd wsinfer
 python -m pip install --editable .[dev]
+pre-commit install
 ```
 
+We use `pre-commit` to automatically run various checks during `git commit`.
+
 # Citation
 
 If you find our work useful, please cite [our paper](https://doi.org/10.1038/s41698-024-00499-9)!

docs/conf.py

@@ -7,6 +7,7 @@
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
 import os
+
 import wsinfer
 
 project = "WSInfer"

docs/installing.rst

@@ -84,6 +84,9 @@ Clone the GitHub repository and install the package in editable mode with the :c
     git clone https://github.com/SBU-BMI/wsinfer.git
     cd wsinfer
     python -m pip install --editable .[dev]
+    pre-commit install
+
+We use :code:`pre-commit` to automatically run various checks during :code:`git commit`.
 
 
 Supported slide backends

pyproject.toml

@@ -28,6 +28,7 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Topic :: Scientific/Engineering :: Artificial Intelligence",
     "Topic :: Scientific/Engineering :: Image Recognition",
     "Topic :: Scientific/Engineering :: Medical Science Apps.",
@@ -62,15 +63,16 @@ dynamic = ["version"]
 [project.optional-dependencies]
 dev = [
     "black",
-    "flake8",
     "geojson",
-    "isort",
     "mypy",
+    "pandas-stubs",
+    "pre-commit",
     "pytest",
+    "ruff",
     "tiffslide",
+    "types-jsonschema",
     "types-Pillow",
     "types-tqdm",
-    "Flake8-pyproject",
 ]
 docs = [
     "pydata-sphinx-theme",
@@ -81,6 +83,7 @@ docs = [
 ]
 openslide = ["openslide-python"]
 qupath = ["paquo"]
+all = ["wsinfer[dev,docs,openslide,qupath]"]
 
 [project.urls]
 Homepage = "https://wsinfer.readthedocs.io"
@@ -97,11 +100,8 @@ wsinfer = ["py.typed", "schemas/*.json"]
 [tool.setuptools.packages.find]
 include = ["wsinfer*"]
 
-# Flake8-pyproject (https://pypi.org/project/Flake8-pyproject/)
-[tool.flake8]
-max-line-length = 88
-extend-ignore = ['E203', 'E701']
-exclude = "wsinfer/_version.py"
+[tool.setuptools_scm]
+write_to = "wsinfer/_version.py"
 
 [tool.mypy]
 disallow_untyped_defs = true
@@ -115,24 +115,20 @@ show_error_codes = true
 [[tool.mypy.overrides]]
 module = [
     "h5py",
-    "cv2",
     "geojson",
     "torchvision.*",
     "openslide",
-    "pandas",
-    "safetensors.*",
-    "scipy.stats",
     "shapely.*",
-    "skimage.morphology",
-    "tifffile",
     "zarr.storage",
-    "paquo.*"
 ]
-ignore_missing_imports = "True"
+ignore_missing_imports = true
 
-[tool.setuptools_scm]
-write_to = "wsinfer/_version.py"
+[tool.ruff]
+extend-exclude = ["_version.py", "scripts/"]
+
+[tool.ruff.lint]
+select = ["E4", "E7", "E9", "F", "B", "I"]
+unfixable = ["B"]
 
-[tool.isort]
-profile = "black"
-force_single_line = "True"
+[tool.ruff.lint.isort]
+force-single-line = true

tests/test_all.py

@@ -155,7 +155,7 @@ def test_cli_run_with_registered_models(
         isinstance(geojson_row["id"], str)
         assert geojson_row["geometry"]["type"] == "Polygon"
     res = []
-    for i, prob_col in enumerate(prob_cols):
+    for prob_col in prob_cols:
         res.append(
             np.array(
                 [dd["properties"]["measurements"][prob_col] for dd in d["features"]]
@@ -167,8 +167,8 @@ def test_cli_run_with_registered_models(
 
     # Check the coordinate values.
     for df_row, geojson_row in zip(df.itertuples(), d["features"]):
-        maxx = df_row.minx + df_row.width
-        maxy = df_row.miny + df_row.height
+        maxx = df_row.minx + df_row.width  # type: ignore
+        maxy = df_row.miny + df_row.height  # type: ignore
         df_coords = [
             [maxx, df_row.miny],
             [maxx, maxy],
@@ -301,7 +301,7 @@ def test_cli_run_model_and_config(tmp_path: Path) -> None:
 @pytest.mark.xfail
 def test_convert_to_sbu() -> None:
     # TODO: create a synthetic output and then convert it. Check that it is valid.
-    assert False
+    raise AssertionError()
 
 
 @pytest.mark.parametrize(

wsinfer/cli/convert_csv_to_sbubmi.py

@@ -143,11 +143,10 @@ def row_to_json(row: pd.Series) -> dict[str, Any]:
 
     # Write heatmap JSON lines file.
     df = pd.read_csv(input)
-    features = df.apply(row_to_json, axis=1)
-    features = features.tolist()
-    features = (json.dumps(row) for row in features)
+    features = df.apply(row_to_json, axis=1).tolist()
+    features_gen = (json.dumps(row) for row in features)
     with open(output_heatmap, "w") as f:
-        f.writelines(line + "\n" for line in features)
+        f.writelines(line + "\n" for line in features_gen)
 
     # Write meta file.
     meta_dict = {

wsinfer/errors.py

@@ -11,7 +11,8 @@ class UnknownArchitectureError(WsinferException):
     """Architecture is unknown and cannot be found."""
 
 
-class WholeSlideImageDirectoryNotFound(WsinferException, FileNotFoundError): ...
+class WholeSlideImageDirectoryNotFound(WsinferException, FileNotFoundError):
+    ...
 
 
 class DuplicateFilePrefixesFound(WsinferException):
@@ -22,19 +23,24 @@ class DuplicateFilePrefixesFound(WsinferException):
     """
 
 
-class WholeSlideImagesNotFound(WsinferException, FileNotFoundError): ...
+class WholeSlideImagesNotFound(WsinferException, FileNotFoundError):
+    ...
 
 
-class ResultsDirectoryNotFound(WsinferException, FileNotFoundError): ...
+class ResultsDirectoryNotFound(WsinferException, FileNotFoundError):
+    ...
 
 
-class PatchDirectoryNotFound(WsinferException, FileNotFoundError): ...
+class PatchDirectoryNotFound(WsinferException, FileNotFoundError):
+    ...
 
 
-class CannotReadSpacing(WsinferException): ...
+class CannotReadSpacing(WsinferException):
+    ...
 
 
-class NoBackendException(WsinferException): ...
+class NoBackendException(WsinferException):
+    ...
 
 
 class BackendNotAvailable(WsinferException):

wsinfer/modellib/models.py

@@ -11,7 +11,8 @@
 
 
 @dataclasses.dataclass
-class LocalModelTorchScript(Model): ...
+class LocalModelTorchScript(Model):
+    ...
 
 
 def get_registered_model(name: str) -> HFModelTorchScript:
@@ -61,7 +62,7 @@ def jit_compile(
         try:
             return torch.compile(model)
         except Exception:
-            warnings.warn(w)
+            warnings.warn(w, stacklevel=1)
             return noncompiled
     # For pytorch 1.x, use torch.jit.script.
     else:
@@ -70,7 +71,7 @@ def jit_compile(
             with torch.no_grad():
                 mjit(test_input)
         except Exception:
-            warnings.warn(w)
+            warnings.warn(w, stacklevel=1)
             return noncompiled
         # Now that we have scripted the model, try to optimize it further. If that
         # fails, return the scripted model.

wsinfer/qupath.py

@@ -5,7 +5,9 @@
 from pathlib import Path
 
 try:
+    from paquo.images import QuPathPathObjectHierarchy
     from paquo.projects import QuPathProject
+    from paquo.projects import QuPathProjectImageEntry
 
     HAS_PAQUO = True
 except Exception:
@@ -25,8 +27,16 @@ def add_image_and_geojson(
             print(f"Unable to find features key:: {e}")
 
     entry = qupath_proj.add_image(image_path)
+    if not isinstance(entry, QuPathProjectImageEntry):
+        print("!!!!!!!!!!!!!!!!!!!!!")
+        print(
+            "Runtime error, please contact developer, explaining that the entry when"
+            " adding an image returns a list of image entry objects."
+        )
+        return
     try:
-        entry.hierarchy.load_geojson(geojson_features)
+        hierarchy: QuPathPathObjectHierarchy = entry.hierarchy
+        hierarchy.load_geojson(geojson_features)
     except Exception as e:
         print(f"Failed to run load_geojson function with error:: {e}")
 

wsinfer/wsi.py

@@ -24,7 +24,7 @@
     # Test that OpenSlide object exists. If it doesn't, an error will be thrown and
     # caught. For some reason, it is possible that openslide-python can be installed
     # but the OpenSlide object (and other openslide things) are not available.
-    openslide.OpenSlide
+    openslide.OpenSlide  # noqa: B018
     HAS_OPENSLIDE = True
     logger.debug("Imported openslide")
 except Exception as err:
@@ -47,11 +47,13 @@
 
 
 @overload
-def set_backend(name: Literal["openslide"]) -> type[openslide.OpenSlide]: ...
+def set_backend(name: Literal["openslide"]) -> type[openslide.OpenSlide]:
+    ...
 
 
 @overload
-def set_backend(name: Literal["tiffslide"]) -> type[tiffslide.TiffSlide]: ...
+def set_backend(name: Literal["tiffslide"]) -> type[tiffslide.TiffSlide]:
+    ...
 
 
 def set_backend(