From da824cd692e3f625dba8581b42209bac23773624 Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Sat, 10 Aug 2024 20:27:24 +0200 Subject: [PATCH] chore: improve docs + CI --- .github/workflows/python_build.yml | 6 ++++-- python/deltalake/schema.py | 5 +++-- python/deltalake/table.py | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python_build.yml b/.github/workflows/python_build.yml index e7a139a5ac..d4ed2d4f7d 100644 --- a/.github/workflows/python_build.yml +++ b/.github/workflows/python_build.yml @@ -31,7 +31,7 @@ jobs: run: make check-rust test-minimal: - name: Python Build (Python 3.8 PyArrow latest) + name: Python Build (Python 3.8 PyArrow 16.0.0) runs-on: ubuntu-latest env: RUSTFLAGS: "-C debuginfo=line-tables-only" @@ -49,7 +49,9 @@ jobs: run: | python -m venv venv source venv/bin/activate - make develop + make setup + # Install minimum PyArrow version + pip install -e .[pandas,devel] pyarrow==16.0.0 env: RUSTFLAGS: "-C debuginfo=line-tables-only" diff --git a/python/deltalake/schema.py b/python/deltalake/schema.py index 06d9c4690d..8bc5c7e155 100644 --- a/python/deltalake/schema.py +++ b/python/deltalake/schema.py @@ -42,8 +42,9 @@ def _convert_pa_schema_to_delta( Args schema: Source schema - large_dtypes: If True, the pyarrow schema is casted to large_dtypes - large_view_passthrough: If True, String/Binary/Lists are passed through as is + schema_conversion_mode: large mode will cast all string/binary/list to the large version arrow types, normal mode + keeps the normal version of the types. Passthrough mode keeps string/binary/list flavored types in their original + version, whether that is view/large/normal. """ dtype_map = { pa.uint8(): pa.int8(), diff --git a/python/deltalake/table.py b/python/deltalake/table.py index 69e61259b6..5cd0d252cf 100644 --- a/python/deltalake/table.py +++ b/python/deltalake/table.py @@ -875,7 +875,7 @@ def merge( target_alias: Optional[str] = None, error_on_type_mismatch: bool = True, writer_properties: Optional[WriterProperties] = None, - large_dtypes: bool = False, + large_dtypes: Optional[bool] = None, custom_metadata: Optional[Dict[str, str]] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None, ) -> "TableMerger":