Merge pull request #267 from zodb/issue264

Remove the umysqldb driver. Fixes #264

.travis.yml

@@ -62,7 +62,6 @@ install:
   - pip install -U "pylint>=1.7.1"
   - pip install -U 'cffi; platform_python_implementation=="CPython"'
   - pip install -U -e ".[test,all_tested_drivers]"
-  - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then pip install git+https://github.com/NextThought/umysqldb.git#egg=umysqldb; fi
   - .travis/postgres.sh
   - .travis/mysql.sh
 

CHANGES.rst

@@ -119,6 +119,9 @@
 - Deprecate the option ``shared-blob-dir``. Shared blob dirs prevent
   using parallel commits when blobs are part of a transaction.
 
+- Remove the 'umysqldb' driver option. This driver exhibited failures
+  with row-level locking used for parallel commits. See :issue:`264`.
+
 3.0a3 (2019-06-26)
 ==================
 

docs/db-specific-options.rst

@@ -116,32 +116,6 @@ driver
       preferred). It is compatible with gevent if gevent's
       monkey-patching is used.
 
-    umysqldb
-      Deprecated. A C-based driver that builds on PyMySQL. It is compatible with
-      gevent if monkey-patching is used, but only works on CPython 2.
-      It does not require the MySQL client development libraries but
-      uses a project called ``umysql`` to communicate with the server
-      using only sockets.
-
-      .. caution::
-
-         This driver contains many known bugs and will be removed
-         for RelStorage 3.0.
-
-      .. note:: Make sure the server has a
-          ``max_allowed_packet`` setting no larger than 16MB. Also
-          make sure that RelStorage's ``blob-chunk-size`` is less than
-          16MB as well.
-
-      .. note:: `This fork of umysqldb
-           <https://github.com/NextThought/umysqldb.git>`_ is
-           recommended. The ``full-buffer`` branch of `this ultramysql
-           fork
-           <https://github.com/NextThought/ultramysql/tree/full-buffer>`_
-           is also recommended if you encounter strange MySQL packet
-           errors.
-
-
     Py MySQL Connector/Python
       This is the `official client
       <https://dev.mysql.com/doc/connector-python/en/>`_ provided by

docs/install.rst

@@ -30,10 +30,10 @@ database.
 
 On CPython2, install psycopg2 2.8+, mysqlclient 1.4+, or cx_Oracle
 5.2+ (but use caution with 5.2.1+); PyMySQL 0.7, MySQL
-Connector/Python 8.0.16 and umysql are also tested to work as is
-pg8000.
+Connector/Python 8.0.16 is also tested to work as is pg8000.
 
-.. note:: umysql support is deprecated and will be removed.
+.. note:: umysql support was removed in RelStorage 3.0. Use 'gevent
+          MySQLdb' instead.
 
 For CPython3, install psycopg2, mysqlclient 1.4+, or cx_Oracle;
 PyMySQL, MySQL Connector/Python  and pg8000 are also known to work.
@@ -74,8 +74,7 @@ bold** are the recommended adapter installed with the extra.
    +==========+=====================+=====================+==============+
    | CPython2 | 1. **mysqlclient**  |  1. **psycopg2**    | **cx_Oracle**|
    |          | 2. PyMySQL          |  2. pg8000          |              |
-   |          | 3. umysqldb         |                     |              |
-   |          | 4. MySQL Connector  |                     |              |
+   |          | 3. MySQL Connector  |                     |              |
    |          |                     |                     |              |
    +----------+---------------------+---------------------+--------------+
    | CPython3 | 1. **mysqlclient**  |  1. **psycopg2**    | **cx_Oracle**|
@@ -89,7 +88,7 @@ bold** are the recommended adapter installed with the extra.
 
 mysqlclient can be used with gevent by explicitly choosing a
 gevent-aware driver. PyMySQL, MySQL Connector/Python (without its C
-extension), pg8000 and umysql are compatible (cooperative) with gevent
+extension), and pg8000 are compatible (cooperative) with gevent
 when the system is monkey-patched.
 
 For additional details and warnings, see the "driver" section for each database in

docs/relstorage-options.rst

@@ -155,9 +155,7 @@ blob-chunk-size
            then that value is later reduced, it won't be possible to
            download blobs that exceed that value.
 
-           The driver may also influence this. For example, umysqldb
-           needs a 1.3MB ``max_allowed_packet`` to send multiple 1MB
-           chunks.
+           The driver may also influence this.
 
         On PostgreSQL and Oracle, this value is used as the memory
         buffer size for blob upload and download operations.

src/relstorage/adapters/mysql/drivers/__init__.py

@@ -43,5 +43,5 @@ def cursor(self, conn):
 
 implement_db_driver_options(
     __name__,
-    'mysqlconnector', 'mysqldb', 'pymysql', 'umysqldb'
+    'mysqlconnector', 'mysqldb', 'pymysql',
 )

src/relstorage/adapters/mysql/oidallocator.py

@@ -61,6 +61,12 @@ def __init__(self, disconnected_exception):
     # FROM tbl_name (without a WHERE clause) in autocommit mode, the
     # sequence starts over for all storage engines except InnoDB and
     # MyISAM."
+    #
+    # Regarding `TRUNCATE TABLE`:
+    #
+    # "Any AUTO_INCREMENT value is reset to its start value. This is
+    # true even for MyISAM and InnoDB, which normally do not reuse
+    # sequence values."
     def set_min_oid(self, cursor, oid):
         """Ensure the next OID is at least the given OID."""
         n = (oid + 15) // 16
@@ -76,10 +82,14 @@ def new_oids(self, cursor):
         # cursor.connection.insert_id(), which was specific to MySQLdb)
         n = cursor.lastrowid
 
-        # At least in one setup (gevent/umysqldb/pymysql/mysql 5.5)
-        # we have observed cursor.lastrowid to be None.
         if n is None:
-            raise self.disconnected_exception("Invalid return for lastrowid")
+            # In the past, in one setup (gevent/umysqldb/pymysql/mysql 5.5)
+            # we have observed cursor.lastrowid to be None. We don't support that
+            # driver anymore, but we leave this here just in case.
+            raise self.disconnected_exception(
+                "Invalid return for lastrowid. "
+                "Please report this to RelStorage."
+            )
 
         if n % 1000 == 0:
             # Clean out previously generated OIDs.

src/relstorage/adapters/tests/test_interfaces.py

@@ -38,7 +38,7 @@ def test_DriverNotAvailableError_str(self, klass=interfaces.DriverNotAvailableEr
                 s
             )
             self.assertIn(
-                " 'umysqldb' (Module: 'umysqldb'; Available:",
+                " 'MySQLdb' (Module: 'MySQLdb'; Available:",
                 s
             )
 

src/relstorage/tests/testmysql.py

@@ -98,10 +98,9 @@ def __init__(self):
 
     def _compute_large_blob_size(self, use_small_blobs):
         # MySQL is limited to the blob_chunk_size as there is no
-        # native blob streaming support. (Note: this depends on
-        # the max_allowed_packet size on the server as well as the driver; both
-        # values default to 1MB. But umysqldb needs 1.3MB max_allowed_packet size
-        # to send multiple 1MB chunks. So keep it small.)
+        # native blob streaming support. (Note: this depends on the
+        # max_allowed_packet size on the server as well as the driver;
+        # both values default to 1MB. So keep it small.)
         return Options().blob_chunk_size
 
     def _make_check_class_HistoryFreeRelStorageTests(self, bases, name):
@@ -111,14 +110,8 @@ def check16MObject(self):
             # `max_allowed_packet`, you probably need to increase it.
             # JAM uses 64M.
             # http://dev.mysql.com/doc/refman/5.7/en/packet-too-large.html
+            bases[0].check16MObject(self)
 
-            # This fails if the driver is umysqldb.
-            try:
-                bases[0].check16MObject(self)
-            except Exception as e:
-                if e.args == (0, 'Query too big'):
-                    raise unittest.SkipTest("Fails with umysqldb")
-                raise
         return self._default_make_check_class(bases, name, klass_dict={
             check16MObject.__name__: check16MObject
         })