Perf: Enable cache, fix value casting for MySQL and MariaDB and add plugin for Engine creation

.github/workflows/test_and_publish.yml

@@ -100,6 +100,7 @@ jobs:
     # Install MariaDB
     - name: Install MariaDB and SpatiaLite
       run: |
+        sudo apt-get update
         sudo apt-get install -y mariadb-server mariadb-client libsqlite3-mod-spatialite libgdal-dev gdal-bin rasterio
 
     # Config PostgreSQL

doc/admin.rst

@@ -33,8 +33,13 @@ MySQL/MariadDB-specific objects
    :private-members:
    :show-inheritance:
 
+.. automodule:: geoalchemy2.admin.dialects.mariadb
+   :members:
+   :private-members:
+   :show-inheritance:
+
 SQLite-specific objects
----------------------------
+-----------------------
 
 .. automodule:: geoalchemy2.admin.dialects.sqlite
    :members:

doc/core_tutorial.rst

@@ -21,7 +21,11 @@ For this tutorial we will use a PostGIS 2 database. To connect we use
 SQLAlchemy's ``create_engine()`` function::
 
     >>> from sqlalchemy import create_engine
-    >>> engine = create_engine('postgresql://gis:gis@localhost/gis', echo=True)
+    >>> engine = create_engine(
+    ...     'postgresql://gis:gis@localhost/gis',
+    ...     echo=True,
+    ...     plugins=["geoalchemy2"],
+    ... )
 
 In this example the name of the database, the database user, and the database
 password, is ``gis``.
@@ -30,6 +34,11 @@ The ``echo`` flag is a shortcut to setting up SQLAlchemy logging, which is
 accomplished via Python's standard logging module. With it is enabled, we'll
 see all the generated SQL produced.
 
+The ``plugins`` argument adds some event listeners to adapt the behavior of
+``GeoAlchemy2`` to the dialect. This is not mandatory but if the plugin is not
+loaded, then the listeners will have to be added to the engine manually (see an
+example in :ref:`spatialite_dialect`).
+
 The return value of ``create_engine`` is an ``Engine`` object, which
 represents the core interface to the database.
 
@@ -84,7 +93,10 @@ be registered into SQLAlchemy, even if it is not used explicitly.
 
     >>> from geoalchemy2 import Geometry  # <= not used but must be imported
     >>> from sqlalchemy import create_engine, MetaData
-    >>> engine = create_engine("postgresql://myuser:mypass@mydb.host.tld/mydbname")
+    >>> engine = create_engine(
+    ...     "postgresql://myuser:mypass@mydb.host.tld/mydbname",
+    ...     plugins=["geoalchemy2"]
+    ... )
     >>> meta = MetaData()
     >>> meta.reflect(bind=engine)
 

doc/dialect_specific_features.rst

@@ -0,0 +1,13 @@
+.. _dialect_specific_features:
+
+Dialect specific features
+=========================
+
+Several dialects handle spatial data in different ways. ``GeoAlchemy2`` tries to hide these
+differences but sometimes manual tweaks are needed.
+
+.. toctree::
+   :maxdepth: 1
+
+   mysql_mariadb_dialect
+   spatialite_dialect

doc/index.rst

@@ -87,7 +87,7 @@ system. If you're new to GeoAlchemy 2 start with this.
 
    orm_tutorial
    core_tutorial
-   spatialite_tutorial
+   dialect_specific_features
 
 
 Gallery
@@ -121,6 +121,7 @@ Reference Documentation
    :maxdepth: 1
 
    admin
+   plugin
    types
    elements
    spatial_functions

doc/mysql_mariadb_dialect.rst

@@ -0,0 +1,43 @@
+.. _mysql_mariadb_dialect:
+
+MySQL / MariaDB Tutorial
+========================
+
+GeoAlchemy 2's main target is PostGIS. But GeoAlchemy 2 also supports MySQL and MariaDB.
+This tutorial describes how to use GeoAlchemy 2 with these dialects.
+
+.. _mysql_mariadb_connect:
+
+Connect to the DB
+-----------------
+
+Just like when using PostGIS connecting to a MySQL or MariaDB database requires an ``Engine``.
+An engine for these dialects can be created in two ways. Using the plugin provided by
+``GeoAlchemy2`` (see :ref:`plugin` for more details)::
+
+    >>> from sqlalchemy import create_engine
+    >>> engine = create_engine(
+    ...     "mysql://user:password@host:port/dbname",
+    ...     echo=True,
+    ...     plugins=["geoalchemy2"]
+    ... )
+
+The call to ``create_engine`` creates an engine bound to the database given in the URL. After that
+a ``before_cursor_execute`` listener is registered on the engine (see
+:func:`geoalchemy2.admin.dialects.mysql.before_cursor_execute` and
+:func:`geoalchemy2.admin.dialects.mariadb.before_cursor_execute`). The listener is responsible for
+converting the parameters passed to query in the proper format, which is often a necessary operation
+for using these dialects, though it depends on the driver used. If the driver does not require such
+conversion, it is possible to disable this feature with the URL parameter
+``geoalchemy2_before_cursor_execute_mysql_convert`` or
+``geoalchemy2_before_cursor_execute_mariadb_convert``, depending on the dialect used.
+
+
+It is also possible to create a raw engine and attach the listener manually::
+
+    >>> from geoalchemy2.admin.dialects.mysql import before_cursor_execute
+    >>> from sqlalchemy import create_engine
+    >>> from sqlalchemy.event import listen
+    >>>
+    >>> engine = create_engine("mysql://user:password@host:port/dbname", echo=True)
+    >>> listen(engine, "before_cursor_execute", before_cursor_execute)

doc/orm_tutorial.rst

@@ -20,7 +20,11 @@ For this tutorial we will use a PostGIS 2 database. To connect we use
 SQLAlchemy's ``create_engine()`` function::
 
     >>> from sqlalchemy import create_engine
-    >>> engine = create_engine('postgresql://gis:gis@localhost/gis', echo=True)
+    >>> engine = create_engine(
+    ...     'postgresql://gis:gis@localhost/gis',
+    ...     echo=True,
+    ...     plugins=["geoalchemy2"]
+    ... )
 
 In this example the name of the database, the database user, and the database
 password, is ``gis``.
@@ -29,6 +33,11 @@ The ``echo`` flag is a shortcut to setting up SQLAlchemy logging, which is
 accomplished via Python's standard logging module. With it is enabled, we'll
 see all the generated SQL produced.
 
+The ``plugins`` argument adds some event listeners to adapt the behavior of
+``GeoAlchemy2`` to the dialect. This is not mandatory but if the plugin is not
+loaded, then the listeners will have to be added to the engine manually (see an
+example in :ref:`spatialite_dialect`).
+
 The return value of ``create_engine`` is an ``Engine`` object, which
 represents the core interface to the database.
 
@@ -128,7 +137,9 @@ Add New Objects
 
 To persist our ``Lake`` object, we ``add()`` it to the ``Session``::
 
+    >>> lake = Lake(name="Majeur", geom="POLYGON((0 0,1 0,1 1,0 1,0 0))")
     >>> session.add(lake)
+    >>> session.commit()
 
 At this point the ``lake`` object has been added to the ``Session``, but no SQL
 has been issued to the database. The object is in a *pending* state. To persist
@@ -237,6 +248,9 @@ We can also apply relationship functions to
 ``session.scalar`` allows executing a clause and returning a scalar
 value (a boolean value in this case).
 
+The value ``True`` indicates that the lake "Garde" does intersects the ``LINESTRING(2 1,4 1)``
+geometry. See the SpatiaLite SQL functions reference list for more information.
+
 The GeoAlchemy functions all start with ``ST_``. Operators are also called as
 functions, but the function names don't include the ``ST_`` prefix. As an
 example let's use PostGIS' ``&&`` operator, which allows testing

doc/plugin.rst

@@ -0,0 +1,9 @@
+.. _plugin:
+
+Plugin
+======
+
+.. automodule:: geoalchemy2.admin.plugin
+   :members:
+   :private-members:
+   :show-inheritance:

doc/spatialite_tutorial.rst → doc/spatialite_dialect.rst

@@ -1,4 +1,4 @@
-.. _spatialite_tutorial:
+.. _spatialite_dialect:
 
 SpatiaLite Tutorial
 ===================
@@ -12,8 +12,18 @@ the :ref:`orm_tutorial`, which you may want to read first.
 Connect to the DB
 -----------------
 
-Just like when using PostGIS connecting to a SpatiaLite database requires an ``Engine``. This is how
-you create one for SpatiaLite::
+Just like when using PostGIS connecting to a SpatiaLite database requires an ``Engine``. An engine
+for the SpatiaLite dialect can be created in two ways. Using the plugin provided by
+``GeoAlchemy2`` (see :ref:`plugin` for more details)::
+
+    >>> from sqlalchemy import create_engine
+    >>> engine = create_engine(
+    ...     "sqlite:///gis.db",
+    ...     echo=True,
+    ...     plugins=["geoalchemy2"]
+    ... )
+
+Or by attaching the listeners manually::
 
     >>> from geoalchemy2 import load_spatialite
     >>> from sqlalchemy import create_engine
@@ -70,108 +80,6 @@ From the user point of view this works in the same way as with PostGIS. The diff
 internally the ``RecoverGeometryColumn`` and ``DiscardGeometryColumn`` management functions will be
 used for the creation and removal of the geometry column.
 
-Create the Table in the Database
---------------------------------
-
-We can now create the ``lake`` table in the ``gis.db`` database::
-
-    >>> Lake.__table__.create(engine)
-
-If we wanted to drop the table we'd use::
-
-    >>> Lake.__table__.drop(engine)
-
-There's nothing specific to SpatiaLite here.
-
-Create a Session
-----------------
-
-When using the SQLAlchemy ORM the ORM interacts with the database through a ``Session``.
-
-    >>> from sqlalchemy.orm import sessionmaker
-    >>> Session = sessionmaker(bind=engine)
-    >>> session = Session()
-
-The session is associated with our SpatiaLite ``Engine``. Again, there's nothing
-specific to SpatiaLite here.
-
-Add New Objects
----------------
-
-We can now create and insert new ``Lake`` objects into the database, the same way we'd
-do it using GeoAlchemy 2 with PostGIS.
-
-::
-
-    >>> lake = Lake(name="Majeur", geom="POLYGON((0 0,1 0,1 1,0 1,0 0))")
-    >>> session.add(lake)
-    >>> session.commit()
-
-We can now query the database for ``Majeur``::
-
-    >>> our_lake = session.query(Lake).filter_by(name="Majeur").first()
-    >>> our_lake.name
-    u"Majeur"
-    >>> our_lake.geom
-    <WKBElement at 0x9af594c; "0103000000010000000500000000000000000000000000000000000000000000000000f03f0000000000000000000000000000f03f000000000000f03f0000000000000000000000000000f03f00000000000000000000000000000000">
-    >>> our_lake.id
-    1
-
-Let's add more lakes::
-
-    >>> session.add_all([
-    ...     Lake(name="Garde", geom="POLYGON((1 0,3 0,3 2,1 2,1 0))"),
-    ...     Lake(name="Orta", geom="POLYGON((3 0,6 0,6 3,3 3,3 0))")
-    ... ])
-    >>> session.commit()
-
-Query
------
-
-Let's make a simple, non-spatial, query::
-
-    >>> query = session.query(Lake).order_by(Lake.name)
-    >>> for lake in query:
-    ...     print(lake.name)
-    ...
-    Garde
-    Majeur
-    Orta
-
-Now a spatial query::
-
-    >>> from geolachemy2 import WKTElement
-    >>> query = session.query(Lake).filter(
-    ...             func.ST_Contains(Lake.geom, WKTElement("POINT(4 1)")))
-    ...
-    >>> for lake in query:
-    ...     print(lake.name)
-    ...
-    Orta
-
-Here's another spatial query, using ``ST_Intersects`` this time::
-
-    >>> query = session.query(Lake).filter(
-    ...             Lake.geom.ST_Intersects(WKTElement("LINESTRING(2 1,4 1)")))
-    ...
-    >>> for lake in query:
-    ...     print(lake.name)
-    ...
-    Garde
-    Orta
-
-We can also apply relationship functions to :class:`geoalchemy2.elements.WKBElement`. For example::
-
-    >>> lake = session.query(Lake).filter_by(name="Garde").one()
-    >>> print(session.scalar(lake.geom.ST_Intersects(WKTElement("LINESTRING(2 1,4 1)"))))
-    1
-
-``session.scalar`` allows executing a clause and returning a scalar value (an integer value in this
-case).
-
-The value ``1`` indicates that the lake "Garde" does intersects the ``LINESTRING(2 1,4 1)``
-geometry. See the SpatiaLite SQL functions reference list for more information.
-
 Function mapping
 ----------------
 
@@ -201,7 +109,7 @@ GeoPackage format
 Starting from the version ``4.2`` of Spatialite, it is possible to use GeoPackage files as DB
 containers. GeoAlchemy 2 is able to handle most of the GeoPackage features automatically if the
 GeoPackage dialect is used (i.e. the DB URL starts with ``gpkg:///``) and the SpatiaLite extension
-is loaded. Usually, this extension should be loaded using the ``load_spatialite_gpkg`` listener::
+is loaded. Usually, this extension should be loaded using the the ``GeoAlchemy2`` plugin (see :ref:`connect <spatialite_connect>` section) or by attaching the ``load_spatialite_gpkg`` listener to the engine::
 
     >>> from geoalchemy2 import load_spatialite_gpkg
     >>> from sqlalchemy import create_engine

geoalchemy2/__init__.py

@@ -8,6 +8,7 @@
 from geoalchemy2 import types  # noqa
 from geoalchemy2.admin.dialects.geopackage import load_spatialite_gpkg  # noqa
 from geoalchemy2.admin.dialects.sqlite import load_spatialite  # noqa
+from geoalchemy2.admin.plugin import GeoEngine  # noqa
 from geoalchemy2.elements import CompositeElement  # noqa
 from geoalchemy2.elements import RasterElement  # noqa
 from geoalchemy2.elements import WKBElement  # noqa
@@ -50,6 +51,7 @@
     "__version__",
     "ArgumentError",
     "CompositeElement",
+    "GeoEngine",
     "Geography",
     "Geometry",
     "Raster",

geoalchemy2/admin/__init__.py

@@ -20,7 +20,7 @@ def select_dialect(dialect_name):
     known_dialects = {
         "geopackage": dialects.geopackage,
         "mysql": dialects.mysql,
-        "mariadb": dialects.mysql,
+        "mariadb": dialects.mariadb,
         "postgresql": dialects.postgresql,
         "sqlite": dialects.sqlite,
     }
@@ -53,7 +53,7 @@ def after_drop(table, bind, **kw):
     @event.listens_for(Column, "after_parent_attach")
     def after_parent_attach(column, table):
         """Automatically add spatial indexes."""
-        if not isinstance(table, Table):
+        if not isinstance(table, Table):  # pragma: no cover
             # For old versions of SQLAlchemy, subqueries might trigger the after_parent_attach event
             # with a selectable as table, so we want to skip this case.
             return
@@ -75,7 +75,7 @@ def after_parent_attach(column, table):
         try:
             if column.type._spatial_index_reflected:
                 return
-        except AttributeError:
+        except AttributeError:  # pragma: no cover
             pass
 
         kwargs = {
@@ -98,7 +98,7 @@ def after_parent_attach(column, table):
         )
 
     @event.listens_for(Table, "column_reflect")
-    def _reflect_geometry_column(inspector, table, column_info):
+    def column_reflect(inspector, table, column_info):
         select_dialect(inspector.bind.dialect.name).reflect_geometry_column(
             inspector, table, column_info
         )

geoalchemy2/admin/dialects/__init__.py

@@ -2,6 +2,7 @@
 
 from geoalchemy2.admin.dialects import common  # noqa
 from geoalchemy2.admin.dialects import geopackage  # noqa
+from geoalchemy2.admin.dialects import mariadb  # noqa
 from geoalchemy2.admin.dialects import mysql  # noqa
 from geoalchemy2.admin.dialects import postgresql  # noqa
 from geoalchemy2.admin.dialects import sqlite  # noqa