From 15538b9b42ba3d0d20ffd287bb4cc463ff8dfb1d Mon Sep 17 00:00:00 2001 From: Adrian Ehrsam <5270024+aersam@users.noreply.github.com> Date: Mon, 4 Mar 2024 11:08:50 +0100 Subject: [PATCH] fix: fix ruff and mypy version and do formatting (#2240) # Description As requested by @ion-elgreco in #2229 , we should fix the formatter versions # Related Issue(s) # Documentation --- docs/src/python/check_constraints.py | 4 +++- python/deltalake/table.py | 6 +++--- python/deltalake/writer.py | 21 ++++++++----------- python/docs/source/_ext/edit_on_github.py | 6 +++--- python/pyproject.toml | 4 ++-- .../test_write_to_pyspark.py | 1 + .../test_writer_readable.py | 1 + 7 files changed, 22 insertions(+), 21 deletions(-) diff --git a/docs/src/python/check_constraints.py b/docs/src/python/check_constraints.py index 16fb8bf374..1bfa62d970 100644 --- a/docs/src/python/check_constraints.py +++ b/docs/src/python/check_constraints.py @@ -13,9 +13,11 @@ def add_constraint(): def add_data(): # --8<-- [start:add_data] - from deltalake import write_deltalake + from deltalake import write_deltalake, DeltaTable import pandas as pd + dt = DeltaTable("../rust/tests/data/simple_table") + df = pd.DataFrame({"id": [-1]}) write_deltalake(dt, df, mode="append", engine="rust") # _internal.DeltaProtocolError: Invariant violations: ["Check or Invariant (id > 0) violated by value in row: [-1]"] diff --git a/python/deltalake/table.py b/python/deltalake/table.py index d80aa8632f..86fffb2a55 100644 --- a/python/deltalake/table.py +++ b/python/deltalake/table.py @@ -1277,9 +1277,9 @@ def __init__( self.not_matched_insert_updates: Optional[List[Dict[str, str]]] = None self.not_matched_insert_predicate: Optional[List[Optional[str]]] = None self.not_matched_by_source_update_updates: Optional[List[Dict[str, str]]] = None - self.not_matched_by_source_update_predicate: Optional[ - List[Optional[str]] - ] = None + self.not_matched_by_source_update_predicate: Optional[List[Optional[str]]] = ( + None + ) self.not_matched_by_source_delete_predicate: Optional[List[str]] = None self.not_matched_by_source_delete_all: Optional[bool] = None diff --git a/python/deltalake/writer.py b/python/deltalake/writer.py index df76ded806..6ebc496436 100644 --- a/python/deltalake/writer.py +++ b/python/deltalake/writer.py @@ -100,8 +100,7 @@ def write_deltalake( large_dtypes: bool = ..., engine: Literal["pyarrow"] = ..., custom_metadata: Optional[Dict[str, str]] = ..., -) -> None: - ... +) -> None: ... @overload @@ -128,8 +127,7 @@ def write_deltalake( engine: Literal["rust"], writer_properties: WriterProperties = ..., custom_metadata: Optional[Dict[str, str]] = ..., -) -> None: - ... +) -> None: ... @overload @@ -157,8 +155,7 @@ def write_deltalake( engine: Literal["rust"], writer_properties: WriterProperties = ..., custom_metadata: Optional[Dict[str, str]] = ..., -) -> None: - ... +) -> None: ... def write_deltalake( @@ -421,12 +418,12 @@ def check_data_is_aligned_with_partition_filtering( ) -> None: if table is None: return - existed_partitions: FrozenSet[ - FrozenSet[Tuple[str, Optional[str]]] - ] = table._table.get_active_partitions() - allowed_partitions: FrozenSet[ - FrozenSet[Tuple[str, Optional[str]]] - ] = table._table.get_active_partitions(partition_filters) + existed_partitions: FrozenSet[FrozenSet[Tuple[str, Optional[str]]]] = ( + table._table.get_active_partitions() + ) + allowed_partitions: FrozenSet[FrozenSet[Tuple[str, Optional[str]]]] = ( + table._table.get_active_partitions(partition_filters) + ) partition_values = pa.RecordBatch.from_arrays( [ batch.column(column_name) diff --git a/python/docs/source/_ext/edit_on_github.py b/python/docs/source/_ext/edit_on_github.py index f7188f189a..241560877c 100644 --- a/python/docs/source/_ext/edit_on_github.py +++ b/python/docs/source/_ext/edit_on_github.py @@ -38,9 +38,9 @@ def html_page_context(app, pagename, templatename, context, doctree): context["display_github"] = True context["github_user"] = app.config.edit_on_github_project.split("/")[0] context["github_repo"] = app.config.edit_on_github_project.split("/")[1] - context[ - "github_version" - ] = f"{app.config.edit_on_github_branch}/{app.config.page_source_prefix}/" + context["github_version"] = ( + f"{app.config.edit_on_github_branch}/{app.config.page_source_prefix}/" + ) def setup(app): diff --git a/python/pyproject.toml b/python/pyproject.toml index e9fc7389af..9b74760948 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -27,8 +27,8 @@ pandas = [ "pandas" ] devel = [ - "mypy", - "ruff>=0.1.5", + "mypy~=1.8.0", + "ruff~=0.3.0", "packaging>=20", "pytest", "pytest-mock", diff --git a/python/tests/pyspark_integration/test_write_to_pyspark.py b/python/tests/pyspark_integration/test_write_to_pyspark.py index 8418f587ca..5cf6490a62 100644 --- a/python/tests/pyspark_integration/test_write_to_pyspark.py +++ b/python/tests/pyspark_integration/test_write_to_pyspark.py @@ -1,4 +1,5 @@ """Tests that deltalake(delta-rs) can write to tables written by PySpark.""" + import pathlib import pyarrow as pa diff --git a/python/tests/pyspark_integration/test_writer_readable.py b/python/tests/pyspark_integration/test_writer_readable.py index ea555074b8..3ade57c6e9 100644 --- a/python/tests/pyspark_integration/test_writer_readable.py +++ b/python/tests/pyspark_integration/test_writer_readable.py @@ -1,4 +1,5 @@ """Test that pyspark can read tables written by deltalake(delta-rs).""" + import pathlib import pyarrow as pa