Improve MariaDB support (#524)

* Improve MariaDB support

* Fix pre-commit config

* Fix CI and minor improvements

---------

Co-authored-by: Tom Cook <tom.cook@veea.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/test_and_publish.yml

@@ -54,28 +54,53 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+      mysql:
+        image: mysql:latest
+        ports:
+          - 3307:3306
+        env:
+          MYSQL_USER: gis
+          MYSQL_PASSWORD: gis
+          MYSQL_DATABASE: gis
+          MYSQL_ROOT_PASSWORD: gis
+        # Set health checks to wait until MySQL has started
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+      mariadb:
+        image: mariadb:latest
+        ports:
+          - 3308:3306
+        env:
+          MARIADB_USER: gis
+          MARIADB_PASSWORD: gis
+          MARIADB_DATABASE: gis
+          MARIADB_ROOT_PASSWORD: gis
+        # Set health checks to wait until MariaDB has started
+        options: >-
+          --health-cmd="healthcheck.sh --connect --innodb_initialized"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+
 
     steps:
 
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
     - uses: actions/checkout@v4
 
-    # Setup Conda for Python and Pypy
-    - name: Install Conda environment with Micromamba
-      uses: mamba-org/setup-micromamba@v1
+    # Setup Python
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
       with:
-        environment-name: test_${{ matrix.python-version.flag }}
-        cache-environment: true
-        create-args: >-
-          ${{ matrix.python-version.pkg_name }}
-          libgdal
-          libspatialite==5.0.1
-          pyproj
-        condarc: |
-          channels:
-            - conda-forge
-            - defaults
-          channel_priority: strict
+        python-version: ${{ matrix.python-version.flag }}
+
+    # Install MariaDB
+    - name: Install MariaDB and SpatiaLite
+      run: |
+        sudo apt-get install -y mariadb-server mariadb-client libsqlite3-mod-spatialite libgdal-dev gdal-bin rasterio
 
     # Config PostgreSQL
     - name: Configure PostgreSQL
@@ -95,13 +120,17 @@ jobs:
         # Drop PostGIS Topology extension to "gis" database
         psql -h localhost -p 5432 -U gis -d gis -c 'DROP EXTENSION IF EXISTS postgis_topology;'
 
-    # Setup MySQL
-    - name: Set up MySQL
+    # Check MySQL
+    - name: Check MySQL
+      run: |
+        mysql --user=gis --password=gis --host=127.0.0.1 -P 3307 -e "SELECT VERSION();"
+        mysql --user=root --password=gis --host=127.0.0.1 -P 3307 -e "GRANT ALL PRIVILEGES ON *.* TO 'gis'@'%' WITH GRANT OPTION;"
+
+    # Check MariaDB
+    - name: Check MariaDB
       run: |
-        sudo systemctl start mysql
-        sudo mysql --user=root --password=root --host=127.0.0.1 -e "CREATE USER 'gis'@'%' IDENTIFIED BY 'gis';"
-        sudo mysql --user=root --password=root --host=127.0.0.1 -e "GRANT ALL PRIVILEGES ON *.* TO 'gis'@'%' WITH GRANT OPTION;"
-        mysql --user=gis --password=gis -e "CREATE DATABASE gis;"
+        mysql --user=gis --password=gis --host=127.0.0.1 -P 3308 -e "SELECT VERSION();"
+        mysql --user=root --password=gis --host=127.0.0.1 -P 3308 -e "GRANT ALL PRIVILEGES ON *.* TO 'gis'@'%' WITH GRANT OPTION;"
 
     # Check python version
     - name: Display Python version
@@ -114,16 +143,19 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip setuptools
-        pip install tox-gh-actions
+        python -m pip install tox-gh-actions
 
     # Run the test suite
     - name: Run the tests
       env:
-        SPATIALITE_LIBRARY_PATH: /home/runner/micromamba/envs/test_${{ matrix.python-version.flag }}/lib/mod_spatialite.so
-        PROJ_LIB: /home/runner/micromamba/envs/test_${{ matrix.python-version.flag }}/share/proj
+        SPATIALITE_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu/mod_spatialite.so
         COVERAGE_FILE: .coverage
-        PYTEST_MYSQL_DB_URL: mysql://gis:gis@127.0.0.1/gis
+        PYTEST_MYSQL_DB_URL: mysql://gis:gis@127.0.0.1:3307/gis
+        PYTEST_MARIADB_DB_URL: mariadb://gis:gis@127.0.0.1:3308/gis
       run: |
+        if [[ ${{ matrix.python-version.flag }} == 'pypy3.8' ]]; then
+          export PYTEST_ADDOPTS='--ignore=tests/gallery/test_insert_raster.py'
+        fi;
         # Run the unit test suite with SQLAlchemy=1.4.* and then with the latest version of SQLAlchemy
         tox -vv
 

.pre-commit-config.yaml

@@ -1,5 +1,3 @@
-default_language_version:
-  python: python3.8
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.6.0

doc/admin.rst

@@ -25,8 +25,8 @@ PostgreSQL-specific objects
    :private-members:
    :show-inheritance:
 
-MySQL-specific objects
----------------------------
+MySQL/MariadDB-specific objects
+-------------------------------
 
 .. automodule:: geoalchemy2.admin.dialects.mysql
    :members:

doc/index.rst

@@ -17,7 +17,7 @@ GeoAlchemy 2 also supports the following dialects:
 * `GeoPackage <http://www.geopackage.org/spec/>`_
 
 Note that using GeoAlchemy 2 with these dialects may require some specific configuration on the
-application side.
+application side. It also may not be optimal for performance.
 
 GeoAlchemy 2 aims to be simpler than its predecessor, `GeoAlchemy
 <https://pypi.python.org/pypi/GeoAlchemy>`_. Simpler to use, and simpler

geoalchemy2/admin/dialects/mysql.py

@@ -8,6 +8,9 @@
 from geoalchemy2.admin.dialects.common import _check_spatial_type
 from geoalchemy2.admin.dialects.common import _spatial_idx_name
 from geoalchemy2.admin.dialects.common import setup_create_drop
+from geoalchemy2.elements import WKBElement
+from geoalchemy2.elements import WKTElement
+from geoalchemy2.shape import to_shape
 from geoalchemy2.types import Geography
 from geoalchemy2.types import Geometry
 
@@ -31,11 +34,16 @@ def reflect_geometry_column(inspector, table, column_info):
     column_name = column_info.get("name")
     schema = table.schema or inspector.default_schema_name
 
+    if inspector.dialect.name == "mariadb":
+        select_srid = "-1, "
+    else:
+        select_srid = "SRS_ID, "
+
     # Check geometry type, SRID and if the column is nullable
-    geometry_type_query = """SELECT DATA_TYPE, SRS_ID, IS_NULLABLE
+    geometry_type_query = """SELECT DATA_TYPE, {}IS_NULLABLE
         FROM INFORMATION_SCHEMA.COLUMNS
         WHERE TABLE_NAME = '{}' and COLUMN_NAME = '{}'""".format(
-        table.name, column_name
+        select_srid, table.name, column_name
     )
     if schema is not None:
         geometry_type_query += """ and table_schema = '{}'""".format(schema)
@@ -176,25 +184,85 @@ def _compile_GeomFromWKB_MySql(element, compiler, **kw):
         return "{}({})".format(element.identifier, compiled)
 
 
+def _compile_GeomFromText_MariaDB(element, compiler, **kw):
+    element.identifier = "ST_GeomFromText"
+    compiled = compiler.process(element.clauses, **kw)
+    try:
+        clauses = list(element.clauses)
+        data_element = WKTElement(clauses[0].value)
+        srid = max(0, data_element.srid)
+        if srid <= 0:
+            srid = max(0, element.type.srid)
+        if len(clauses) > 1 and srid > 0:
+            clauses[1].value = srid
+    except Exception:
+        srid = max(0, element.type.srid)
+
+    if srid > 0:
+        res = "{}({}, {})".format(element.identifier, compiled, srid)
+    else:
+        res = "{}({})".format(element.identifier, compiled)
+    return res
+
+
+def _compile_GeomFromWKB_MariaDB(element, compiler, **kw):
+    element.identifier = "ST_GeomFromText"
+
+    try:
+        clauses = list(element.clauses)
+        data_element = WKBElement(clauses[0].value)
+        srid = max(0, data_element.srid)
+        if srid <= 0:
+            srid = max(0, element.type.srid)
+        clauses[0].value = to_shape(data_element).wkt.encode("utf-8")
+        if len(clauses) > 1 and srid > 0:
+            clauses[1].value = srid
+    except Exception:
+        srid = max(0, element.type.srid)
+    compiled = compiler.process(element.clauses, **kw)
+
+    if srid > 0:
+        res = "{}({}, {})".format(element.identifier, compiled, srid)
+    else:
+        res = "{}({})".format(element.identifier, compiled)
+    return res
+
+
 @compiles(functions.ST_GeomFromText, "mysql")  # type: ignore
-@compiles(functions.ST_GeomFromText, "mariadb")  # type: ignore
 def _MySQL_ST_GeomFromText(element, compiler, **kw):
     return _compile_GeomFromText_MySql(element, compiler, **kw)
 
 
 @compiles(functions.ST_GeomFromEWKT, "mysql")  # type: ignore
-@compiles(functions.ST_GeomFromEWKT, "mariadb")  # type: ignore
 def _MySQL_ST_GeomFromEWKT(element, compiler, **kw):
     return _compile_GeomFromText_MySql(element, compiler, **kw)
 
 
+@compiles(functions.ST_GeomFromText, "mariadb")  # type: ignore
+def _MariaDB_ST_GeomFromText(element, compiler, **kw):
+    return _compile_GeomFromText_MariaDB(element, compiler, **kw)
+
+
+@compiles(functions.ST_GeomFromEWKT, "mariadb")  # type: ignore
+def _MariaDB_ST_GeomFromEWKT(element, compiler, **kw):
+    return _compile_GeomFromText_MariaDB(element, compiler, **kw)
+
+
 @compiles(functions.ST_GeomFromWKB, "mysql")  # type: ignore
-@compiles(functions.ST_GeomFromWKB, "mariadb")  # type: ignore
 def _MySQL_ST_GeomFromWKB(element, compiler, **kw):
     return _compile_GeomFromWKB_MySql(element, compiler, **kw)
 
 
 @compiles(functions.ST_GeomFromEWKB, "mysql")  # type: ignore
-@compiles(functions.ST_GeomFromEWKB, "mariadb")  # type: ignore
 def _MySQL_ST_GeomFromEWKB(element, compiler, **kw):
     return _compile_GeomFromWKB_MySql(element, compiler, **kw)
+
+
+@compiles(functions.ST_GeomFromWKB, "mariadb")  # type: ignore
+def _MariaDB_ST_GeomFromWKB(element, compiler, **kw):
+    return _compile_GeomFromWKB_MariaDB(element, compiler, **kw)
+
+
+@compiles(functions.ST_GeomFromEWKB, "mariadb")  # type: ignore
+def _MariaDB_ST_GeomFromEWKB(element, compiler, **kw):
+    return _compile_GeomFromWKB_MariaDB(element, compiler, **kw)

geoalchemy2/types/__init__.py

@@ -40,7 +40,7 @@ def select_dialect(dialect_name):
     known_dialects = {
         "geopackage": dialects.geopackage,
         "mysql": dialects.mysql,
-        "mariadb": dialects.mysql,
+        "mariadb": dialects.mariadb,
         "postgresql": dialects.postgresql,
         "sqlite": dialects.sqlite,
     }
@@ -198,17 +198,17 @@ def check_ctor_args(geometry_type, srid, dimension, use_typmod, nullable):
         return geometry_type, srid
 
 
-@compiles(_GISType, "mariadb")
 @compiles(_GISType, "mysql")
-def get_col_spec(self, *args, **kwargs):
+@compiles(_GISType, "mariadb")
+def get_col_spec_mysql(self, compiler, *args, **kwargs):
     if self.geometry_type is not None:
         spec = "%s" % self.geometry_type
     else:
         spec = "GEOMETRY"
 
     if not self.nullable or self.spatial_index:
         spec += " NOT NULL"
-    if self.srid > 0:
+    if self.srid > 0 and compiler.dialect.name != "mariadb":
         spec += " SRID %d" % self.srid
     return spec
 

geoalchemy2/types/dialects/__init__.py

@@ -2,6 +2,7 @@
 
 from geoalchemy2.types.dialects import common  # noqa
 from geoalchemy2.types.dialects import geopackage  # noqa
+from geoalchemy2.types.dialects import mariadb  # noqa
 from geoalchemy2.types.dialects import mysql  # noqa
 from geoalchemy2.types.dialects import postgresql  # noqa
 from geoalchemy2.types.dialects import sqlite  # noqa

geoalchemy2/types/dialects/mariadb.py

@@ -0,0 +1,47 @@
+"""This module defines specific functions for MySQL dialect."""
+
+from geoalchemy2.elements import WKBElement
+from geoalchemy2.elements import WKTElement
+from geoalchemy2.elements import _SpatialElement
+from geoalchemy2.exc import ArgumentError
+from geoalchemy2.shape import to_shape
+
+
+def bind_processor_process(spatial_type, bindvalue):
+    if isinstance(bindvalue, str):
+        wkt_match = WKTElement._REMOVE_SRID.match(bindvalue)
+        srid = wkt_match.group(2)
+        try:
+            if srid is not None:
+                srid = int(srid)
+        except (ValueError, TypeError):  # pragma: no cover
+            raise ArgumentError(
+                f"The SRID ({srid}) of the supplied value can not be casted to integer"
+            )
+
+        if srid is not None and srid != spatial_type.srid:
+            raise ArgumentError(
+                f"The SRID ({srid}) of the supplied value is different "
+                f"from the one of the column ({spatial_type.srid})"
+            )
+        return wkt_match.group(3)
+
+    if (
+        isinstance(bindvalue, _SpatialElement)
+        and bindvalue.srid != -1
+        and bindvalue.srid != spatial_type.srid
+    ):
+        raise ArgumentError(
+            f"The SRID ({bindvalue.srid}) of the supplied value is different "
+            f"from the one of the column ({spatial_type.srid})"
+        )
+
+    if isinstance(bindvalue, WKTElement):
+        bindvalue = bindvalue.as_wkt()
+        if bindvalue.srid <= 0:
+            bindvalue.srid = spatial_type.srid
+        return bindvalue
+    elif isinstance(bindvalue, WKBElement):
+        # With MariaDB we use Shapely to convert the WKBElement to an EWKT string
+        return to_shape(bindvalue).wkt
+    return bindvalue

pyproject.toml

@@ -5,6 +5,9 @@ requires = [
     "wheel",
     "setuptools_scm[toml]>=3.4",
 ]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
 
 # BLACK
 [tool.black]

requirements.txt

@@ -6,4 +6,4 @@ pytest
 pytest-cov
 pytest-html
 pytest-mypy
-rasterio
+rasterio;implementation_name!='pypy'

test_container/Dockerfile

@@ -4,8 +4,8 @@ COPY ./helpers/install_requirements.sh /
 RUN /install_requirements.sh
 
 COPY ./helpers/init_postgres.sh /
-env PGDATA="/var/lib/postgresql/data"
-env POSTGRES_PATH="/usr/lib/postgresql/16"
+ENV PGDATA="/var/lib/postgresql/data"
+ENV POSTGRES_PATH="/usr/lib/postgresql/16"
 RUN su postgres -c /init_postgres.sh
 
 ENV SPATIALITE_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu/mod_spatialite.so"

test_container/Dockerfile_mariadb

@@ -0,0 +1,19 @@
+FROM ubuntu:22.04
+
+COPY ./helpers/install_requirements.sh /
+RUN /install_requirements.sh
+
+RUN apt-get update; apt-get install -y mariadb-server mariadb-client; rm -rf /var/lib/apt/lists/*;
+
+COPY ./helpers/init_postgres.sh /
+ENV PGDATA="/var/lib/postgresql/data"
+ENV POSTGRES_PATH="/usr/lib/postgresql/16"
+RUN su postgres -c /init_postgres.sh
+
+ENV SPATIALITE_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu/mod_spatialite.so"
+
+COPY ./helpers/init_mariadb.sh /
+RUN /init_mariadb.sh
+
+COPY ./helpers/entrypoint.sh /
+ENTRYPOINT ["/entrypoint.sh"]