From 0f20ccea5919f9e0c4fc8c430e1d34a4aefbba0f Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Wed, 11 Sep 2024 14:25:54 -0400 Subject: [PATCH 1/5] ci: test examples --- .github/workflows/ibis-backends.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ibis-backends.yml b/.github/workflows/ibis-backends.yml index 4a030a23891e..80da162cbcf4 100644 --- a/.github/workflows/ibis-backends.yml +++ b/.github/workflows/ibis-backends.yml @@ -68,7 +68,7 @@ jobs: run: pip install 'poetry==1.8.3' - name: install ibis - run: poetry install --without dev --without docs --extras bigquery + run: poetry install --without dev --without docs --extras "bigquery examples" - uses: extractions/setup-just@v2 env: @@ -468,7 +468,7 @@ jobs: run: pip install 'poetry==1.8.3' - name: install ibis - run: poetry install --without dev --without docs --extras "${{ join(matrix.backend.extras, ' ') }}" + run: poetry install --without dev --without docs --extras "${{ join(matrix.backend.extras, ' ') }} examples" - name: install deps for broken avro-python setup if: matrix.backend.name == 'flink' @@ -663,7 +663,7 @@ jobs: run: poetry lock --no-update - name: install ibis - run: poetry install --without dev --without docs --extras "${{ join(matrix.backend.extras, ' ') }}" + run: poetry install --without dev --without docs --extras "${{ join(matrix.backend.extras, ' ') }} examples" - name: run tests run: just ci-check -m ${{ matrix.backend.name }} --numprocesses auto --dist=loadgroup @@ -749,7 +749,7 @@ jobs: run: poetry lock --no-update - name: install ibis - run: poetry install --without dev --without docs --extras pyspark + run: poetry install --without dev --without docs --extras "pyspark examples" - name: install delta-spark if: matrix.pyspark-version == '3.5' From ede8e5daf99348e51d6f67c25f233b061b23a6e2 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Wed, 11 Sep 2024 14:28:25 -0400 Subject: [PATCH 2/5] chore: no need for examples in simple bigquery tests --- .github/workflows/ibis-backends.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ibis-backends.yml b/.github/workflows/ibis-backends.yml index 80da162cbcf4..c1639326fac6 100644 --- a/.github/workflows/ibis-backends.yml +++ b/.github/workflows/ibis-backends.yml @@ -68,7 +68,7 @@ jobs: run: pip install 'poetry==1.8.3' - name: install ibis - run: poetry install --without dev --without docs --extras "bigquery examples" + run: poetry install --without dev --without docs --extras bigquery - uses: extractions/setup-just@v2 env: From 539f19315b221fbafaa35886f1f94051d733bbf8 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Wed, 11 Sep 2024 14:29:09 -0400 Subject: [PATCH 3/5] chore: add examples extra to clouds --- .github/workflows/ibis-backends-cloud.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ibis-backends-cloud.yml b/.github/workflows/ibis-backends-cloud.yml index 7984abaf7849..00e5f8031475 100644 --- a/.github/workflows/ibis-backends-cloud.yml +++ b/.github/workflows/ibis-backends-cloud.yml @@ -107,7 +107,7 @@ jobs: run: poetry add snowflake-snowpark-python --python="==${{ steps.install_python.outputs.python-version }}" - name: install ibis - run: poetry install --without dev --without docs --extras "${{ join(matrix.backend.extras, ' ') }}" + run: poetry install --without dev --without docs --extras "${{ join(matrix.backend.extras, ' ') }} examples" - uses: extractions/setup-just@v2 env: From b98a4b1105408bd0291ff74ee5d3db6c935b8a7c Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Wed, 11 Sep 2024 14:30:39 -0400 Subject: [PATCH 4/5] fix(datafusion): import pyarrow at the top level --- ibis/backends/datafusion/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ibis/backends/datafusion/__init__.py b/ibis/backends/datafusion/__init__.py index 0c3aa5fcb6a3..ae00eecbdc94 100644 --- a/ibis/backends/datafusion/__init__.py +++ b/ibis/backends/datafusion/__init__.py @@ -9,6 +9,7 @@ import datafusion as df import pyarrow as pa +import pyarrow.dataset as ds import pyarrow_hotfix # noqa: F401 import sqlglot as sg import sqlglot.expressions as sge @@ -372,7 +373,7 @@ def _register( self.con.deregister_table(table_name) self.con.register_record_batches(table_name, [[source]]) return self.table(table_name) - elif isinstance(source, pa.dataset.Dataset): + elif isinstance(source, ds.Dataset): self.con.deregister_table(table_name) self.con.register_dataset(table_name, source) return self.table(table_name) From a92afacf4efa5b4d650a8046aea378fd67020896 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Wed, 11 Sep 2024 14:31:17 -0400 Subject: [PATCH 5/5] test(datafusion): import pyarrow import into the test --- ibis/backends/datafusion/tests/test_register.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ibis/backends/datafusion/tests/test_register.py b/ibis/backends/datafusion/tests/test_register.py index 16a82973c7fa..e846b6fb4ae9 100644 --- a/ibis/backends/datafusion/tests/test_register.py +++ b/ibis/backends/datafusion/tests/test_register.py @@ -2,7 +2,6 @@ import pandas as pd import pyarrow as pa -import pyarrow.dataset as ds import pytest import ibis @@ -45,6 +44,8 @@ def test_register_batches(conn): def test_register_dataset(conn): + import pyarrow.dataset as ds + tab = pa.table({"x": [1, 2, 3]}) dataset = ds.InMemoryDataset(tab) with pytest.warns(FutureWarning, match="v9.1"):