diff --git a/python/Cargo.toml b/python/Cargo.toml index ddf63c5dba..7f46075dc2 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -51,6 +51,7 @@ requires-dist = [ "pytest; extra == 'devel'", "pytest-mock; extra == 'devel'", "pytest-cov; extra == 'devel'", + "pytest-clarity; extra == 'devel'", "pytest-timeout; extra == 'devel'", "sphinx; extra == 'devel'", "sphinx-rtd-theme; extra == 'devel'", diff --git a/python/Makefile b/python/Makefile index f0ce38e04f..eb9ca75bd6 100644 --- a/python/Makefile +++ b/python/Makefile @@ -1,8 +1,7 @@ .DEFAULT_GOAL := help VENV := venv -PY_SOURCE_DIR := deltalake tests - +MATURIN_VERSION := $(shell awk -F '[ ="]+' '$$1 == "requires" { print $$4 }' pyproject.toml) .PHONY: setup-venv setup-venv: ## Setup the virtualenv @@ -12,7 +11,7 @@ setup-venv: ## Setup the virtualenv .PHONY: setup setup: ## Setup the requirements $(info --- Setup dependencies ---) - pip install maturin==0.10.6 + pip install maturin==$(MATURIN_VERSION) .PHONY: develop develop: setup ## Install Python binding of delta-rs @@ -24,7 +23,7 @@ format: ## Format the code $(info --- Rust format ---) cargo fmt $(info --- Python format ---) - black deltalake tests *.py + black isort . .PHONY: check-rust @@ -37,11 +36,11 @@ check-rust: ## Run check on Rust .PHONY: check-python check-python: ## Run check on Python $(info Check Python isort) - isort --diff --check-only $(PY_SOURCE_DIR) + isort --diff --check-only . $(info Check Python black) - black --check $(PY_SOURCE_DIR) + black --check $(info Check Python mypy) - mypy --config-file setup.cfg $(PY_SOURCE_DIR) + mypy .PHONY: unit-test unit-test: ## Run unit test diff --git a/python/deltalake/__init__.py b/python/deltalake/__init__.py index 1e46701585..8af468f5d5 100644 --- a/python/deltalake/__init__.py +++ b/python/deltalake/__init__.py @@ -1,3 +1,3 @@ -from .deltalake import RawDeltaTable, RawDeltaTableMetaData, rust_core_version +from .deltalake import RawDeltaTable, rust_core_version from .schema import DataType, Field, Schema from .table import DeltaTable, Metadata diff --git a/python/deltalake/table.py b/python/deltalake/table.py index 1120e678f7..3fdfd994c0 100644 --- a/python/deltalake/table.py +++ b/python/deltalake/table.py @@ -10,7 +10,7 @@ if TYPE_CHECKING: import pandas -from .deltalake import RawDeltaTable, RawDeltaTableMetaData +from .deltalake import RawDeltaTable from .schema import Schema, pyarrow_schema_from_json diff --git a/python/mypy.ini b/python/mypy.ini deleted file mode 100644 index 59c262682f..0000000000 --- a/python/mypy.ini +++ /dev/null @@ -1,20 +0,0 @@ -[mypy] -disallow_any_generics = True -disallow_subclassing_any = True -disallow_untyped_calls = True -disallow_untyped_defs = True -disallow_incomplete_defs = True -check_untyped_defs = True -disallow_untyped_decorators = True -no_implicit_optional = True -warn_redundant_casts = True -warn_unused_ignores = True -warn_return_any = False -implicit_reexport = False -strict_equality = True - -[mypy-pyarrow.*] -ignore_missing_imports = True - -[mypy-deltalake.deltalake] -ignore_missing_imports = True \ No newline at end of file diff --git a/python/pyproject.toml b/python/pyproject.toml index f2dd04d3b0..37eff6e172 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,9 +1,29 @@ [build-system] -requires = ["maturin==0.10.6"] +requires = ["maturin==0.11.2"] build-backend = "maturin" +[tool.mypy] +files = "deltalake/*.py" +exclude = "^tests" +mypy_path = "./stubs" +disallow_any_generics = true +disallow_subclassing_any = true +disallow_untyped_calls = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true +disallow_untyped_decorators = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_return_any = false +implicit_reexport = false +strict_equality = true + [tool.isort] profile = "black" +src_paths = ["deltalake", "tests"] [tool.black] +include = "deltalake,tests" exclude = "venv" \ No newline at end of file diff --git a/python/setup.cfg b/python/setup.cfg deleted file mode 100644 index 1d5747a2c3..0000000000 --- a/python/setup.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[flake8] -max-line-length=150 - -[mypy] -ignore_missing_imports = True diff --git a/python/stubs/deltalake/__init__.pyi b/python/stubs/deltalake/__init__.pyi new file mode 100644 index 0000000000..e994c5f3a6 --- /dev/null +++ b/python/stubs/deltalake/__init__.pyi @@ -0,0 +1,3 @@ +from typing import Any + +RawDeltaTableMetadata: Any diff --git a/python/stubs/deltalake/deltalake.pyi b/python/stubs/deltalake/deltalake.pyi new file mode 100644 index 0000000000..d3c5529703 --- /dev/null +++ b/python/stubs/deltalake/deltalake.pyi @@ -0,0 +1,4 @@ +from typing import Any + +RawDeltaTable: Any +rust_core_version: Any \ No newline at end of file diff --git a/python/stubs/pandas/__init__.pyi b/python/stubs/pandas/__init__.pyi new file mode 100644 index 0000000000..be86cb108b --- /dev/null +++ b/python/stubs/pandas/__init__.pyi @@ -0,0 +1,3 @@ +from typing import Any + +DataFrame: Any \ No newline at end of file diff --git a/python/stubs/pyarrow/__init__.pyi b/python/stubs/pyarrow/__init__.pyi new file mode 100644 index 0000000000..ecf969d47f --- /dev/null +++ b/python/stubs/pyarrow/__init__.pyi @@ -0,0 +1,18 @@ +from typing import Any + +Schema: Any +Table: Any +Field: Any +DataType: Any +schema: Any +map_: Any +list_: Any +field: Any +struct: Any +type_for_alias: Any +date32: Any +date64: Any +decimal128: Any +float16: Any +float32: Any +float64: Any diff --git a/python/stubs/pyarrow/dataset.pyi b/python/stubs/pyarrow/dataset.pyi new file mode 100644 index 0000000000..254718661a --- /dev/null +++ b/python/stubs/pyarrow/dataset.pyi @@ -0,0 +1,5 @@ +from typing import Any + +Dataset: Any +dataset: Any +partitioning: Any \ No newline at end of file