From 3c56d85b2fe0e7bfd50fa02701f0b6e40a54172b Mon Sep 17 00:00:00 2001 From: Sundeep Anand Date: Tue, 16 Apr 2024 14:11:17 +0930 Subject: [PATCH 01/12] Temporary fix for geometries with Z in SpatiaLite. Though most of the dialects support geom_type ending with Z in SQL queries, it seems not working with SpatiaLite. Hence replacing Z with a blank. Added a test to cover this behaviour. --- doc/spatialite_tutorial.rst | 2 +- geoalchemy2/types/dialects/sqlite.py | 9 +++++- tests/test_functional_sqlite.py | 41 ++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/doc/spatialite_tutorial.rst b/doc/spatialite_tutorial.rst index ef5bf8b9..0e56ceb6 100644 --- a/doc/spatialite_tutorial.rst +++ b/doc/spatialite_tutorial.rst @@ -54,7 +54,7 @@ Declare a Mapping Now that we have a working connection we can go ahead and create a mapping between a Python class and a database table:: - >>> from sqlalchemy.ext.declarative import declarative_base + >>> from sqlalchemy.orm import declarative_base >>> from sqlalchemy import Column, Integer, String >>> from geoalchemy2 import Geometry >>> diff --git a/geoalchemy2/types/dialects/sqlite.py b/geoalchemy2/types/dialects/sqlite.py index 93d4f0a2..72470ccf 100644 --- a/geoalchemy2/types/dialects/sqlite.py +++ b/geoalchemy2/types/dialects/sqlite.py @@ -13,9 +13,16 @@ def bind_processor_process(spatial_type, bindvalue): else: return "SRID=%d;%s" % (bindvalue.srid, bindvalue.data) elif isinstance(bindvalue, WKBElement): + if bindvalue.srid == -1: + bindvalue.srid = spatial_type.srid # With SpatiaLite we use Shapely to convert the WKBElement to an EWKT string shape = to_shape(bindvalue) - return "SRID=%d;%s" % (bindvalue.srid, shape.wkt) + result = "SRID=%d;%s" % (bindvalue.srid, shape.wkt) + if shape.has_z: + # shapely.wkb.loads returns geom_type with a 'Z', for example, 'LINESTRING Z' + # which is a limitation with SpatiaLite. Hence, a temporary fix. + result = result.replace('Z ', '') + return result elif isinstance(bindvalue, RasterElement): return "%s" % (bindvalue.data) else: diff --git a/tests/test_functional_sqlite.py b/tests/test_functional_sqlite.py index 262e0267..9e61dc9e 100644 --- a/tests/test_functional_sqlite.py +++ b/tests/test_functional_sqlite.py @@ -129,6 +129,47 @@ def test_explicit_schema(self, conn): # Drop the table t.drop(bind=conn) + def test_3d_geometry(self, conn, metadata): + # Define the table + col = Column( + "geom", + Geometry( + geometry_type=None, + srid=4326, + spatial_index=False + ), + nullable=False, + ) + t = Table( + "3d_geom_type", + metadata, + Column("id", Integer, primary_key=True), + col, + ) + + # Create the table + t.create(bind=conn) + + # Should be 'LINESTRING Z (0 0 0, 1 1 1)' + # Read comments at geoalchemy2/types/dialects/sqlite.py#L22 + elements = {"geom": "SRID=4326;LINESTRING (0 0 0, 1 1 1)"} + conn.execute(t.insert(), elements) + + with pytest.raises((IntegrityError, OperationalError)): + with conn.begin_nested(): + # This returns a NULL for the geom field. + conn.execute( + t.insert(), [{"geom": "SRID=4326;LINESTRING Z (0 0 0, 1 1 1)"}] + ) + + results = conn.execute(t.select()) + rows = results.fetchall() + + assert len(rows) == 1 + + # Drop the table + t.drop(bind=conn) + class TestIndex: @pytest.fixture From 1ec380db89caec55bb26be4f1b438ca4de000518 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 Apr 2024 05:14:21 +0000 Subject: [PATCH 02/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- geoalchemy2/types/dialects/sqlite.py | 2 +- tests/test_functional_sqlite.py | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/geoalchemy2/types/dialects/sqlite.py b/geoalchemy2/types/dialects/sqlite.py index 72470ccf..f19f0e8f 100644 --- a/geoalchemy2/types/dialects/sqlite.py +++ b/geoalchemy2/types/dialects/sqlite.py @@ -21,7 +21,7 @@ def bind_processor_process(spatial_type, bindvalue): if shape.has_z: # shapely.wkb.loads returns geom_type with a 'Z', for example, 'LINESTRING Z' # which is a limitation with SpatiaLite. Hence, a temporary fix. - result = result.replace('Z ', '') + result = result.replace("Z ", "") return result elif isinstance(bindvalue, RasterElement): return "%s" % (bindvalue.data) diff --git a/tests/test_functional_sqlite.py b/tests/test_functional_sqlite.py index 9e61dc9e..53f34ef9 100644 --- a/tests/test_functional_sqlite.py +++ b/tests/test_functional_sqlite.py @@ -133,11 +133,7 @@ def test_3d_geometry(self, conn, metadata): # Define the table col = Column( "geom", - Geometry( - geometry_type=None, - srid=4326, - spatial_index=False - ), + Geometry(geometry_type=None, srid=4326, spatial_index=False), nullable=False, ) t = Table( @@ -158,9 +154,7 @@ def test_3d_geometry(self, conn, metadata): with pytest.raises((IntegrityError, OperationalError)): with conn.begin_nested(): # This returns a NULL for the geom field. - conn.execute( - t.insert(), [{"geom": "SRID=4326;LINESTRING Z (0 0 0, 1 1 1)"}] - ) + conn.execute(t.insert(), [{"geom": "SRID=4326;LINESTRING Z (0 0 0, 1 1 1)"}]) results = conn.execute(t.select()) rows = results.fetchall() From 4c49bf3f428f201b33402f4460f37f45904895f8 Mon Sep 17 00:00:00 2001 From: Adrien Berchet Date: Tue, 16 Apr 2024 15:10:39 +0200 Subject: [PATCH 03/12] Add a more complete test and propose an extended fix --- geoalchemy2/elements.py | 1 + geoalchemy2/types/dialects/sqlite.py | 37 ++++++++---- tests/test_functional.py | 85 ++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 11 deletions(-) diff --git a/geoalchemy2/elements.py b/geoalchemy2/elements.py index 6221bc12..2ef55db7 100644 --- a/geoalchemy2/elements.py +++ b/geoalchemy2/elements.py @@ -117,6 +117,7 @@ class WKTElement(_SpatialElement): """ _REMOVE_SRID = re.compile("(SRID=([0-9]+); ?)?(.*)") + SPLIT_WKT_PATTERN = re.compile(r"((SRID=\d+) *; *)?([\w ]+) *(\([\d ,\(\)]+\))") geom_from: str = "ST_GeomFromText" geom_from_extended_version: str = "ST_GeomFromEWKT" diff --git a/geoalchemy2/types/dialects/sqlite.py b/geoalchemy2/types/dialects/sqlite.py index f19f0e8f..cef5617c 100644 --- a/geoalchemy2/types/dialects/sqlite.py +++ b/geoalchemy2/types/dialects/sqlite.py @@ -1,29 +1,44 @@ """This module defines specific functions for SQLite dialect.""" +import re + from geoalchemy2.elements import RasterElement from geoalchemy2.elements import WKBElement from geoalchemy2.elements import WKTElement from geoalchemy2.shape import to_shape +def format_geom_type(wkt, forced_srid=None): + """Format the Geometry type for SQLite.""" + match = re.match(WKTElement.SPLIT_WKT_PATTERN, wkt) + if match is None: + return wkt + _, srid, geom_type, coords = match.groups() + geom_type = geom_type.replace(" ", "") + if geom_type.endswith("M"): + geom_type = geom_type[:-1] + if geom_type.endswith("Z"): + geom_type = geom_type[:-1] + if forced_srid is not None: + srid = f"SRID={forced_srid}" + if srid is not None: + return "%s;%s%s" % (srid, geom_type, coords) + else: + return "%s%s" % (geom_type, coords) + + def bind_processor_process(spatial_type, bindvalue): if isinstance(bindvalue, WKTElement): - if bindvalue.extended: - return "%s" % (bindvalue.data) - else: - return "SRID=%d;%s" % (bindvalue.srid, bindvalue.data) + return format_geom_type(bindvalue.data, forced_srid=bindvalue.srid) elif isinstance(bindvalue, WKBElement): if bindvalue.srid == -1: bindvalue.srid = spatial_type.srid # With SpatiaLite we use Shapely to convert the WKBElement to an EWKT string shape = to_shape(bindvalue) - result = "SRID=%d;%s" % (bindvalue.srid, shape.wkt) - if shape.has_z: - # shapely.wkb.loads returns geom_type with a 'Z', for example, 'LINESTRING Z' - # which is a limitation with SpatiaLite. Hence, a temporary fix. - result = result.replace("Z ", "") - return result + # shapely.wkb.loads returns geom_type with a 'Z', for example, 'LINESTRING Z' + # which is a limitation with SpatiaLite. Hence, a temporary fix. + return format_geom_type(shape.wkt, forced_srid=bindvalue.srid) elif isinstance(bindvalue, RasterElement): return "%s" % (bindvalue.data) else: - return bindvalue + return format_geom_type(bindvalue) diff --git a/tests/test_functional.py b/tests/test_functional.py index 62187e64..1833305a 100644 --- a/tests/test_functional.py +++ b/tests/test_functional.py @@ -275,6 +275,91 @@ def test_insert(self, conn, Lake, setup_tables): srid = conn.execute(row[1].ST_SRID()).scalar() assert srid == 4326 + @pytest.mark.parametrize( + "geom_type,wkt", + [ + pytest.param("POINT", "(1 2)", id="Point"), + pytest.param("POINTZ", "(1 2 3)", id="Point Z"), + pytest.param("LINESTRING", "(1 2, 3 4)", id="LineString"), + pytest.param("LINESTRINGZ", "(1 2 3, 4 5 6)", id="LineString Z"), + pytest.param("POLYGON", "((1 2, 3 4, 5 6, 1 2))", id="Polygon"), + pytest.param("POLYGONZ", "((1 2 3, 4 5 6, 7 8 9, 1 2 3))", id="Polygon Z"), + pytest.param("MULTIPOINT", "(1 2, 3 4)", id="Multi Point"), + pytest.param("MULTIPOINTZ", "(1 2 3, 4 5 6)", id="Multi Point Z"), + pytest.param("MULTILINESTRING", "((1 2, 3 4), (10 20, 30 40))", id="Multi LineString"), + pytest.param( + "MULTILINESTRINGZ", + "((1 2 3, 4 5 6), (10 20 30, 40 50 60))", + id="Multi LineString Z", + ), + pytest.param( + "MULTIPOLYGON", + "(((1 2, 3 4, 5 6, 1 2), (10 20, 30 40, 50 60, 10 20)))", + id="Multi Polygon", + ), + pytest.param( + "MULTIPOLYGONZ", + "(((1 2 3, 4 5 6, 7 8 9, 1 2 3), (10 20 30, 40 50 60, 70 80 90, 10 20 30)))", + id="Multi Polygon Z", + ), + ], + ) + def test_insert_all_geom_types(self, dialect_name, base, conn, metadata, geom_type, wkt): + """Test insertion and selection of all geometry types.""" + ndims = 2 + if "Z" in geom_type[-2:]: + ndims += 1 + if geom_type.endswith("M"): + ndims += 1 + + if ndims > 2 and dialect_name == "mysql": + # Explicitly skip MySQL dialect to show that it can only work with 2D geometries + pytest.skip(reason="MySQL only supports 2D geometry types") + + class GeomTypeTable(base): + __tablename__ = "test_geom_types" + id = Column(Integer, primary_key=True) + geom = Column(Geometry(srid=4326, geometry_type=geom_type, dimension=ndims)) + + metadata.drop_all(bind=conn, checkfirst=True) + metadata.create_all(bind=conn) + + inserted_wkt = f"{geom_type}{wkt}" + + # Use the DB to generate the corresponding raw WKB + raw_wkb = conn.execute( + text("SELECT ST_AsBinary(ST_GeomFromText('{}', 4326))".format(inserted_wkt)) + ).scalar() + + inserted_elements = [ + {"geom": f"SRID=4326;{inserted_wkt}"}, + {"geom": WKTElement(inserted_wkt, srid=4326)}, + {"geom": WKTElement(f"SRID=4326;{inserted_wkt}")}, + ] + if dialect_name not in ["sqlite", "geopackage"]: + inserted_elements.append({"geom": inserted_wkt}) + if dialect_name not in ["sqlite", "geopackage"] or ndims == 2: + inserted_elements.append({"geom": WKBElement(raw_wkb, srid=4326)}) + + # Insert the elements + conn.execute( + GeomTypeTable.__table__.insert(), + inserted_elements, + ) + + # Select the elements + query = select([GeomTypeTable.__table__.c.geom.ST_AsText()]) + results = conn.execute(query) + rows = results.scalars().all() + + # Check that the selected elements are the same as the inputs + for row in rows: + checked_wkt = row.upper().replace(" ", "") + expected_wkt = inserted_wkt.upper().replace(" ", "") + if dialect_name == "mysql" and geom_type == "MULTIPOINT": + checked_wkt = re.sub(r"\((\d+)\)", "\\1", checked_wkt) + assert checked_wkt == expected_wkt + @test_only_with_dialects("postgresql", "sqlite") def test_insert_geom_poi(self, conn, Poi, setup_tables): conn.execute( From e5846824d6307ef9e920f7f02083d49dbb6c3434 Mon Sep 17 00:00:00 2001 From: Adrien Berchet Date: Fri, 19 Apr 2024 13:00:07 +0200 Subject: [PATCH 04/12] Handle M coordinate --- geoalchemy2/types/dialects/sqlite.py | 26 ++++++---- tests/test_functional.py | 72 +++++++++++++++++++++++----- tests/test_functional_sqlite.py | 35 -------------- 3 files changed, 76 insertions(+), 57 deletions(-) diff --git a/geoalchemy2/types/dialects/sqlite.py b/geoalchemy2/types/dialects/sqlite.py index cef5617c..023dff96 100644 --- a/geoalchemy2/types/dialects/sqlite.py +++ b/geoalchemy2/types/dialects/sqlite.py @@ -8,19 +8,19 @@ from geoalchemy2.shape import to_shape -def format_geom_type(wkt, forced_srid=None): +def format_geom_type(wkt, default_srid=None): """Format the Geometry type for SQLite.""" match = re.match(WKTElement.SPLIT_WKT_PATTERN, wkt) if match is None: return wkt _, srid, geom_type, coords = match.groups() geom_type = geom_type.replace(" ", "") - if geom_type.endswith("M"): - geom_type = geom_type[:-1] + if geom_type.endswith("ZM"): + geom_type = geom_type[:-2] if geom_type.endswith("Z"): geom_type = geom_type[:-1] - if forced_srid is not None: - srid = f"SRID={forced_srid}" + if srid is None and default_srid is not None: + srid = f"SRID={default_srid}" if srid is not None: return "%s;%s%s" % (srid, geom_type, coords) else: @@ -29,16 +29,22 @@ def format_geom_type(wkt, forced_srid=None): def bind_processor_process(spatial_type, bindvalue): if isinstance(bindvalue, WKTElement): - return format_geom_type(bindvalue.data, forced_srid=bindvalue.srid) + return format_geom_type( + bindvalue.data, + default_srid=bindvalue.srid if bindvalue.srid >= 0 else spatial_type.srid, + ) elif isinstance(bindvalue, WKBElement): - if bindvalue.srid == -1: - bindvalue.srid = spatial_type.srid # With SpatiaLite we use Shapely to convert the WKBElement to an EWKT string shape = to_shape(bindvalue) # shapely.wkb.loads returns geom_type with a 'Z', for example, 'LINESTRING Z' # which is a limitation with SpatiaLite. Hence, a temporary fix. - return format_geom_type(shape.wkt, forced_srid=bindvalue.srid) + res = format_geom_type( + shape.wkt, default_srid=bindvalue.srid if bindvalue.srid >= 0 else spatial_type.srid + ) + return res elif isinstance(bindvalue, RasterElement): return "%s" % (bindvalue.data) + elif isinstance(bindvalue, str): + return format_geom_type(bindvalue, default_srid=spatial_type.srid) else: - return format_geom_type(bindvalue) + return bindvalue diff --git a/tests/test_functional.py b/tests/test_functional.py index 1833305a..d0e8299b 100644 --- a/tests/test_functional.py +++ b/tests/test_functional.py @@ -280,28 +280,59 @@ def test_insert(self, conn, Lake, setup_tables): [ pytest.param("POINT", "(1 2)", id="Point"), pytest.param("POINTZ", "(1 2 3)", id="Point Z"), + pytest.param("POINTM", "(1 2 3)", id="Point M"), + pytest.param("POINTZM", "(1 2 3 4)", id="Point ZM"), pytest.param("LINESTRING", "(1 2, 3 4)", id="LineString"), pytest.param("LINESTRINGZ", "(1 2 3, 4 5 6)", id="LineString Z"), + pytest.param("LINESTRINGM", "(1 2 3, 4 5 6)", id="LineString M"), + pytest.param("LINESTRINGZM", "(1 2 3 4, 5 6 7 8)", id="LineString ZM"), pytest.param("POLYGON", "((1 2, 3 4, 5 6, 1 2))", id="Polygon"), pytest.param("POLYGONZ", "((1 2 3, 4 5 6, 7 8 9, 1 2 3))", id="Polygon Z"), + pytest.param("POLYGONM", "((1 2 3, 4 5 6, 7 8 9, 1 2 3))", id="Polygon M"), + pytest.param( + "POLYGONZM", "((1 2 3 4, 5 6 7 8, 9 10 11 12, 1 2 3 4))", id="Polygon ZM" + ), pytest.param("MULTIPOINT", "(1 2, 3 4)", id="Multi Point"), pytest.param("MULTIPOINTZ", "(1 2 3, 4 5 6)", id="Multi Point Z"), + pytest.param("MULTIPOINTM", "(1 2 3, 4 5 6)", id="Multi Point M"), + pytest.param("MULTIPOINTZM", "(1 2 3 4, 5 6 7 8)", id="Multi Point ZM"), pytest.param("MULTILINESTRING", "((1 2, 3 4), (10 20, 30 40))", id="Multi LineString"), pytest.param( "MULTILINESTRINGZ", "((1 2 3, 4 5 6), (10 20 30, 40 50 60))", id="Multi LineString Z", ), + pytest.param( + "MULTILINESTRINGM", + "((1 2 3, 4 5 6), (10 20 30, 40 50 60))", + id="Multi LineString M", + ), + pytest.param( + "MULTILINESTRINGZM", + "((1 2 3 4, 5 6 7 8), (10 20 30 40, 50 60 70 80))", + id="Multi LineString ZM", + ), pytest.param( "MULTIPOLYGON", - "(((1 2, 3 4, 5 6, 1 2), (10 20, 30 40, 50 60, 10 20)))", + "(((1 2, 3 4, 5 6, 1 2)), ((10 20, 30 40, 50 60, 10 20)))", id="Multi Polygon", ), pytest.param( "MULTIPOLYGONZ", - "(((1 2 3, 4 5 6, 7 8 9, 1 2 3), (10 20 30, 40 50 60, 70 80 90, 10 20 30)))", + "(((1 2 3, 4 5 6, 7 8 9, 1 2 3)), ((10 20 30, 40 50 60, 70 80 90, 10 20 30)))", id="Multi Polygon Z", ), + pytest.param( + "MULTIPOLYGONM", + "(((1 2 3, 4 5 6, 7 8 9, 1 2 3)), ((10 20 30, 40 50 60, 70 80 90, 10 20 30)))", + id="Multi Polygon M", + ), + pytest.param( + "MULTIPOLYGONZM", + "(((1 2 3 4, 5 6 7 8, 9 10 11 12, 1 2 3 4))," + " ((10 20 30 40, 50 60 70 80, 90 100 100 120, 10 20 30 40)))", + id="Multi Polygon ZM", + ), ], ) def test_insert_all_geom_types(self, dialect_name, base, conn, metadata, geom_type, wkt): @@ -311,10 +342,13 @@ def test_insert_all_geom_types(self, dialect_name, base, conn, metadata, geom_ty ndims += 1 if geom_type.endswith("M"): ndims += 1 + has_m = True + else: + has_m = False if ndims > 2 and dialect_name == "mysql": # Explicitly skip MySQL dialect to show that it can only work with 2D geometries - pytest.skip(reason="MySQL only supports 2D geometry types") + pytest.xfail(reason="MySQL only supports 2D geometry types") class GeomTypeTable(base): __tablename__ = "test_geom_types" @@ -331,15 +365,17 @@ class GeomTypeTable(base): text("SELECT ST_AsBinary(ST_GeomFromText('{}', 4326))".format(inserted_wkt)) ).scalar() + wkb_elem = WKBElement(raw_wkb, srid=4326) inserted_elements = [ + {"geom": inserted_wkt}, {"geom": f"SRID=4326;{inserted_wkt}"}, {"geom": WKTElement(inserted_wkt, srid=4326)}, {"geom": WKTElement(f"SRID=4326;{inserted_wkt}")}, ] - if dialect_name not in ["sqlite", "geopackage"]: - inserted_elements.append({"geom": inserted_wkt}) - if dialect_name not in ["sqlite", "geopackage"] or ndims == 2: - inserted_elements.append({"geom": WKBElement(raw_wkb, srid=4326)}) + if dialect_name not in ["postgresql", "sqlite"] or not has_m: + # Currently Shapely does not support geometry types with M dimension + inserted_elements.append({"geom": wkb_elem}) + inserted_elements.append({"geom": wkb_elem.as_ewkb()}) # Insert the elements conn.execute( @@ -348,17 +384,29 @@ class GeomTypeTable(base): ) # Select the elements - query = select([GeomTypeTable.__table__.c.geom.ST_AsText()]) + query = select( + [ + GeomTypeTable.__table__.c.id, + GeomTypeTable.__table__.c.geom.ST_AsText(), + GeomTypeTable.__table__.c.geom.ST_SRID(), + ], + ) results = conn.execute(query) - rows = results.scalars().all() + rows = results.all() # Check that the selected elements are the same as the inputs - for row in rows: + for row_id, row, srid in rows: checked_wkt = row.upper().replace(" ", "") expected_wkt = inserted_wkt.upper().replace(" ", "") if dialect_name == "mysql" and geom_type == "MULTIPOINT": checked_wkt = re.sub(r"\((\d+)\)", "\\1", checked_wkt) - assert checked_wkt == expected_wkt + print(row_id, row, srid) + if row_id >= 5 and dialect_name in ["geopackage"] and has_m: + # Currently Shapely does not support geometry types with M dimension + assert checked_wkt != expected_wkt + else: + assert checked_wkt == expected_wkt + assert srid == 4326 @test_only_with_dialects("postgresql", "sqlite") def test_insert_geom_poi(self, conn, Poi, setup_tables): @@ -465,7 +513,7 @@ def test_WKT(self, session, Lake, setup_tables, dialect_name, postgis_version): lake = Lake("LINESTRING(0 0,1 1)") session.add(lake) - if (dialect_name == "postgresql" and postgis_version < 3) or dialect_name == "sqlite": + if dialect_name == "postgresql" and postgis_version < 3: with pytest.raises((DataError, IntegrityError)): session.flush() else: diff --git a/tests/test_functional_sqlite.py b/tests/test_functional_sqlite.py index 53f34ef9..262e0267 100644 --- a/tests/test_functional_sqlite.py +++ b/tests/test_functional_sqlite.py @@ -129,41 +129,6 @@ def test_explicit_schema(self, conn): # Drop the table t.drop(bind=conn) - def test_3d_geometry(self, conn, metadata): - # Define the table - col = Column( - "geom", - Geometry(geometry_type=None, srid=4326, spatial_index=False), - nullable=False, - ) - t = Table( - "3d_geom_type", - metadata, - Column("id", Integer, primary_key=True), - col, - ) - - # Create the table - t.create(bind=conn) - - # Should be 'LINESTRING Z (0 0 0, 1 1 1)' - # Read comments at geoalchemy2/types/dialects/sqlite.py#L22 - elements = {"geom": "SRID=4326;LINESTRING (0 0 0, 1 1 1)"} - conn.execute(t.insert(), elements) - - with pytest.raises((IntegrityError, OperationalError)): - with conn.begin_nested(): - # This returns a NULL for the geom field. - conn.execute(t.insert(), [{"geom": "SRID=4326;LINESTRING Z (0 0 0, 1 1 1)"}]) - - results = conn.execute(t.select()) - rows = results.fetchall() - - assert len(rows) == 1 - - # Drop the table - t.drop(bind=conn) - class TestIndex: @pytest.fixture From 2d81faa99798b4f56ab1b2e888a57f362a5641a7 Mon Sep 17 00:00:00 2001 From: Sundeep Anand Date: Sun, 21 Apr 2024 09:06:04 +0930 Subject: [PATCH 05/12] Change postgis image in test_and_publish.yml to postgis/postgis:16-3.4 Add some formatting for postgresql dialect, MULTIPOINT geom_type. --- .github/workflows/test_and_publish.yml | 2 +- tests/test_functional.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_and_publish.yml b/.github/workflows/test_and_publish.yml index ece59324..4c8adb97 100644 --- a/.github/workflows/test_and_publish.yml +++ b/.github/workflows/test_and_publish.yml @@ -41,7 +41,7 @@ jobs: services: postgres: - image: mdillon/postgis:11 + image: postgis/postgis:16-3.4 env: POSTGRES_DB: gis POSTGRES_PASSWORD: gis diff --git a/tests/test_functional.py b/tests/test_functional.py index d0e8299b..beb20ac4 100644 --- a/tests/test_functional.py +++ b/tests/test_functional.py @@ -358,6 +358,13 @@ class GeomTypeTable(base): metadata.drop_all(bind=conn, checkfirst=True) metadata.create_all(bind=conn) + if dialect_name == 'postgresql' and 'MULTIPOINT' in geom_type: + # formats wkt to wrap tuple elements in brackets, + # for example '(1 2, 3 4)' to '((1 2), (3 4))'. + wkt = "({})".format( + ", ".join(map(lambda x: '({})'.format(x.strip()), wkt[1:-1].split(','))) + ) + inserted_wkt = f"{geom_type}{wkt}" # Use the DB to generate the corresponding raw WKB From f903e71d9348256e6e719f8b14987431c85701e5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 20 Apr 2024 23:36:31 +0000 Subject: [PATCH 06/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_functional.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_functional.py b/tests/test_functional.py index beb20ac4..5bdd6975 100644 --- a/tests/test_functional.py +++ b/tests/test_functional.py @@ -358,11 +358,11 @@ class GeomTypeTable(base): metadata.drop_all(bind=conn, checkfirst=True) metadata.create_all(bind=conn) - if dialect_name == 'postgresql' and 'MULTIPOINT' in geom_type: + if dialect_name == "postgresql" and "MULTIPOINT" in geom_type: # formats wkt to wrap tuple elements in brackets, # for example '(1 2, 3 4)' to '((1 2), (3 4))'. wkt = "({})".format( - ", ".join(map(lambda x: '({})'.format(x.strip()), wkt[1:-1].split(','))) + ", ".join(map(lambda x: "({})".format(x.strip()), wkt[1:-1].split(","))) ) inserted_wkt = f"{geom_type}{wkt}" From c0a5cbb41df9bda834959ad51e5974aca4600d2b Mon Sep 17 00:00:00 2001 From: Sundeep Anand Date: Sun, 21 Apr 2024 09:15:13 +0930 Subject: [PATCH 07/12] Enable postgis_raster extension for postgresql gis db. --- .github/workflows/test_and_publish.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test_and_publish.yml b/.github/workflows/test_and_publish.yml index 4c8adb97..9b7aa7b4 100644 --- a/.github/workflows/test_and_publish.yml +++ b/.github/workflows/test_and_publish.yml @@ -86,6 +86,9 @@ jobs: # Add PostGIS extension to "gis" database psql -h localhost -p 5432 -U gis -d gis -c 'CREATE EXTENSION IF NOT EXISTS postgis;' + # Add PostGISRaster extension to "gis" database + psql -h localhost -p 5432 -U gis -d gis -c 'CREATE EXTENSION IF NOT EXISTS postgis_raster;' + # Drop PostGIS Tiger Geocoder extension to "gis" database psql -h localhost -p 5432 -U gis -d gis -c 'DROP EXTENSION IF EXISTS postgis_tiger_geocoder CASCADE;' From 746e51906236e82184c728df318a2c4ab4c8bc7b Mon Sep 17 00:00:00 2001 From: Adrien Berchet Date: Sun, 21 Apr 2024 20:40:26 +0200 Subject: [PATCH 08/12] Fix docs, tests and docker helpers --- doc/conf.py | 3 +++ geoalchemy2/types/dialects/sqlite.py | 2 +- test_container/Dockerfile | 2 +- test_container/helpers/install_requirements.sh | 10 ++++++---- tests/test_functional.py | 14 ++++---------- 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 1a8783a5..cf10be95 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -60,6 +60,9 @@ # version = release = geoalchemy2.__version__ +# Remove some Sphinx warnings +suppress_warnings = ["config.cache"] + # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # language = None diff --git a/geoalchemy2/types/dialects/sqlite.py b/geoalchemy2/types/dialects/sqlite.py index 023dff96..7aa2cf09 100644 --- a/geoalchemy2/types/dialects/sqlite.py +++ b/geoalchemy2/types/dialects/sqlite.py @@ -17,7 +17,7 @@ def format_geom_type(wkt, default_srid=None): geom_type = geom_type.replace(" ", "") if geom_type.endswith("ZM"): geom_type = geom_type[:-2] - if geom_type.endswith("Z"): + elif geom_type.endswith("Z"): geom_type = geom_type[:-1] if srid is None and default_srid is not None: srid = f"SRID={default_srid}" diff --git a/test_container/Dockerfile b/test_container/Dockerfile index 15a098c9..329235fe 100644 --- a/test_container/Dockerfile +++ b/test_container/Dockerfile @@ -5,7 +5,7 @@ RUN /install_requirements.sh COPY ./helpers/init_postgres.sh / env PGDATA="/var/lib/postgresql/data" -env POSTGRES_PATH="/usr/lib/postgresql/14" +env POSTGRES_PATH="/usr/lib/postgresql/16" RUN su postgres -c /init_postgres.sh ENV SPATIALITE_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu/mod_spatialite.so" diff --git a/test_container/helpers/install_requirements.sh b/test_container/helpers/install_requirements.sh index 5e39df2a..f7319f85 100755 --- a/test_container/helpers/install_requirements.sh +++ b/test_container/helpers/install_requirements.sh @@ -29,9 +29,9 @@ packages=( python3.12-venv # PostgreSQL and PostGIS - postgresql - postgresql-14-postgis-3 - postgresql-14-postgis-3-scripts + postgresql-16 + postgresql-16-postgis-3 + postgresql-16-postgis-3-scripts libpq-dev libgeos-dev @@ -56,8 +56,10 @@ packages=( export DEBIAN_FRONTEND=noninteractive apt-get update -y -apt-get install --no-install-recommends -y software-properties-common gnupg2 +apt-get install --no-install-recommends -y software-properties-common gnupg2 wget add-apt-repository ppa:deadsnakes/ppa +sh -c 'echo "deb https://apt.PostgreSQL.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' +wget --quiet -O - https://www.PostgreSQL.org/media/keys/ACCC4CF8.asc | apt-key add - apt-get update -y apt-get install --no-install-recommends -y "${packages[@]}" diff --git a/tests/test_functional.py b/tests/test_functional.py index 5bdd6975..f69ad75a 100644 --- a/tests/test_functional.py +++ b/tests/test_functional.py @@ -358,13 +358,6 @@ class GeomTypeTable(base): metadata.drop_all(bind=conn, checkfirst=True) metadata.create_all(bind=conn) - if dialect_name == "postgresql" and "MULTIPOINT" in geom_type: - # formats wkt to wrap tuple elements in brackets, - # for example '(1 2, 3 4)' to '((1 2), (3 4))'. - wkt = "({})".format( - ", ".join(map(lambda x: "({})".format(x.strip()), wkt[1:-1].split(","))) - ) - inserted_wkt = f"{geom_type}{wkt}" # Use the DB to generate the corresponding raw WKB @@ -405,9 +398,10 @@ class GeomTypeTable(base): for row_id, row, srid in rows: checked_wkt = row.upper().replace(" ", "") expected_wkt = inserted_wkt.upper().replace(" ", "") - if dialect_name == "mysql" and geom_type == "MULTIPOINT": - checked_wkt = re.sub(r"\((\d+)\)", "\\1", checked_wkt) - print(row_id, row, srid) + if "MULTIPOINT" in geom_type: + # Some dialects return MULTIPOINT geometries with nested parenthesis and others + # do not so we remove them before checking the results + checked_wkt = re.sub(r"\(([0-9]+)\)", "\\1", checked_wkt) if row_id >= 5 and dialect_name in ["geopackage"] and has_m: # Currently Shapely does not support geometry types with M dimension assert checked_wkt != expected_wkt From 8c743c0dc6c04492bfdd7255261582d5441ca019 Mon Sep 17 00:00:00 2001 From: Adrien Berchet Date: Mon, 22 Apr 2024 10:11:17 +0200 Subject: [PATCH 09/12] Fix postgis_raster extension in CI --- .github/workflows/test_and_publish.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_and_publish.yml b/.github/workflows/test_and_publish.yml index 9b7aa7b4..16587f9d 100644 --- a/.github/workflows/test_and_publish.yml +++ b/.github/workflows/test_and_publish.yml @@ -84,13 +84,13 @@ jobs: psql -h localhost -p 5432 -U gis -d gis -c 'CREATE SCHEMA gis;' # Add PostGIS extension to "gis" database - psql -h localhost -p 5432 -U gis -d gis -c 'CREATE EXTENSION IF NOT EXISTS postgis;' + psql -h localhost -p 5432 -U gis -d gis -c 'CREATE EXTENSION IF NOT EXISTS postgis SCHEMA public;' # Add PostGISRaster extension to "gis" database - psql -h localhost -p 5432 -U gis -d gis -c 'CREATE EXTENSION IF NOT EXISTS postgis_raster;' + psql -h localhost -p 5432 -U gis -d gis -c 'CREATE EXTENSION IF NOT EXISTS postgis_raster SCHEMA public;' # Drop PostGIS Tiger Geocoder extension to "gis" database - psql -h localhost -p 5432 -U gis -d gis -c 'DROP EXTENSION IF EXISTS postgis_tiger_geocoder CASCADE;' + psql -h localhost -p 5432 -U gis -d gis -c 'DROP EXTENSION IF EXISTS postgis_tiger_geocoder SCHEMA public CASCADE;' # Setup MySQL - name: Set up MySQL From 9a204168342837798e81e6579091d7f583b5192c Mon Sep 17 00:00:00 2001 From: Sundeep Anand Date: Tue, 23 Apr 2024 01:06:45 +0930 Subject: [PATCH 10/12] Update .github/workflows/test_and_publish.yml Co-authored-by: Adrien Berchet --- .github/workflows/test_and_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_and_publish.yml b/.github/workflows/test_and_publish.yml index 16587f9d..dc6de73e 100644 --- a/.github/workflows/test_and_publish.yml +++ b/.github/workflows/test_and_publish.yml @@ -90,7 +90,7 @@ jobs: psql -h localhost -p 5432 -U gis -d gis -c 'CREATE EXTENSION IF NOT EXISTS postgis_raster SCHEMA public;' # Drop PostGIS Tiger Geocoder extension to "gis" database - psql -h localhost -p 5432 -U gis -d gis -c 'DROP EXTENSION IF EXISTS postgis_tiger_geocoder SCHEMA public CASCADE;' + psql -h localhost -p 5432 -U gis -d gis -c 'DROP EXTENSION IF EXISTS postgis_tiger_geocoder CASCADE;' # Setup MySQL - name: Set up MySQL From a7637c3d81a998dce6ad8a63bef7f0a55f16f0b7 Mon Sep 17 00:00:00 2001 From: Sundeep Anand Date: Tue, 23 Apr 2024 09:55:00 +0930 Subject: [PATCH 11/12] Update .github/workflows/test_and_publish.yml Co-authored-by: Adrien Berchet --- .github/workflows/test_and_publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_and_publish.yml b/.github/workflows/test_and_publish.yml index dc6de73e..22d81a65 100644 --- a/.github/workflows/test_and_publish.yml +++ b/.github/workflows/test_and_publish.yml @@ -89,8 +89,8 @@ jobs: # Add PostGISRaster extension to "gis" database psql -h localhost -p 5432 -U gis -d gis -c 'CREATE EXTENSION IF NOT EXISTS postgis_raster SCHEMA public;' - # Drop PostGIS Tiger Geocoder extension to "gis" database - psql -h localhost -p 5432 -U gis -d gis -c 'DROP EXTENSION IF EXISTS postgis_tiger_geocoder CASCADE;' + # Drop PostGIS Topology extension to "gis" database + psql -h localhost -p 5432 -U gis -d gis -c 'DROP EXTENSION IF EXISTS postgis_topology;' # Setup MySQL - name: Set up MySQL From 32f3886c6d9cd9adc72342161583cc4697cc9756 Mon Sep 17 00:00:00 2001 From: Sundeep Anand Date: Tue, 23 Apr 2024 16:27:45 +0930 Subject: [PATCH 12/12] Update .github/workflows/test_and_publish.yml Co-authored-by: Adrien Berchet --- .github/workflows/test_and_publish.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test_and_publish.yml b/.github/workflows/test_and_publish.yml index 22d81a65..10b1be79 100644 --- a/.github/workflows/test_and_publish.yml +++ b/.github/workflows/test_and_publish.yml @@ -86,6 +86,9 @@ jobs: # Add PostGIS extension to "gis" database psql -h localhost -p 5432 -U gis -d gis -c 'CREATE EXTENSION IF NOT EXISTS postgis SCHEMA public;' + # Drop PostGIS Tiger Geocoder extension to "gis" database + psql -h localhost -p 5432 -U gis -d gis -c 'DROP EXTENSION IF EXISTS postgis_tiger_geocoder CASCADE;' + # Add PostGISRaster extension to "gis" database psql -h localhost -p 5432 -U gis -d gis -c 'CREATE EXTENSION IF NOT EXISTS postgis_raster SCHEMA public;'