From 3baf509bbccc15a2173ad82fe81613916a4a0c9f Mon Sep 17 00:00:00 2001 From: ncclementi Date: Fri, 15 Dec 2023 12:32:57 -0700 Subject: [PATCH] feat(geospatial): update read_geo to support url In the upcoming update of the spatial extension, to be able to read form a url, the httpfs extension is needed. --- ibis/backends/duckdb/__init__.py | 7 ++++++- ibis/backends/duckdb/tests/test_register.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/ibis/backends/duckdb/__init__.py b/ibis/backends/duckdb/__init__.py index 3a82204af5c0..75ab280e3a63 100644 --- a/ibis/backends/duckdb/__init__.py +++ b/ibis/backends/duckdb/__init__.py @@ -528,8 +528,13 @@ def read_geo( # load geospatial extension self.load_extension("spatial") + source = util.normalize_filename(source) + + if source.startswith(("http://", "https://", "s3://")): + self._load_extensions(["httpfs"]) + source_expr = sa.select(sa.literal_column("*")).select_from( - sa.func.st_read(util.normalize_filename(source), _format_kwargs(kwargs)) + sa.func.st_read(source, _format_kwargs(kwargs)) ) view = self._compile_temp_view(table_name, source_expr) diff --git a/ibis/backends/duckdb/tests/test_register.py b/ibis/backends/duckdb/tests/test_register.py index c98afb5d3971..0568ed7712b6 100644 --- a/ibis/backends/duckdb/tests/test_register.py +++ b/ibis/backends/duckdb/tests/test_register.py @@ -108,6 +108,20 @@ def test_read_geo_to_geopandas(con, data_dir): assert isinstance(gdf, gpd.GeoDataFrame) +def test_read_geo_from_url(con, monkeypatch): + loaded_exts = [] + monkeypatch.setattr(con, "_load_extensions", lambda x, **kw: loaded_exts.extend(x)) + + with pytest.raises((sa.exc.OperationalError, sa.exc.ProgrammingError)): + # The read will fail, either because the URL is bogus (which it is) or + # because the current connection doesn't have the spatial extension + # installed and so the call to `st_read` will raise a catalog error. + con.read_geo("https://...") + + assert "spatial" in loaded_exts + assert "httpfs" in loaded_exts + + @pytest.mark.xfail_version( duckdb=["duckdb<0.7.0"], reason="read_json_auto doesn't exist", raises=exc.IbisError )