113 Improve reporting of test failures for CQL tests

python/adelphi/bin/test-adelphi

@@ -0,0 +1,86 @@
+#!python
+
+# Run test suite for some set of Cassandra versions.
+#
+# We implement this as a front-end script because the tox/pytest/unittest chain
+# isn't great about iterating over a suite of test fixtures and re-running the
+# collected tests for each of them.  Rather than fight with the frameworks we
+# manage the fixture iteration manually in this script.  This also has the
+# nice side effect of moving a lot of C* checking/session code out of the test
+# suite, which in turn should allow us to write simpler tests.
+import configparser
+import os
+
+from tests.util.cassandra_util import connectToLocalCassandra
+
+import click
+import docker
+from tenacity import retry, stop_after_attempt, wait_fixed
+import tox
+
+
+# Default C* versions to include in all integration tests
+DEFAULT_CASSANDRA_VERSIONS = ["2.1.22", "2.2.19", "3.0.23", "3.11.9", "4.0-rc1"]
+
+TOX_DEPENDENCIES = """pytest
+    subprocess32 ~= 3.5
+    tenacity ~= 7.0"""
+TOX_CONFIG = "tox.ini"
+
+
+@retry(stop=stop_after_attempt(3), wait=wait_fixed(2))
+def runCassandraContainer(client, version):
+    return client.containers.run(name="adelphi", remove=True, detach=True, ports={9042: 9042}, image="cassandra:{}".format(version))
+
+
+def writeToxIni(version):
+    config = configparser.ConfigParser()
+    config["tox"] = { "envlist": "py2, py3" }
+    envs = {"CASSANDRA_VERSION": version}
+    config["testenv"] = {"deps": TOX_DEPENDENCIES, \
+        "commands": "pytest {posargs}", \
+        "setenv": "CASSANDRA_VERSION = {}".format(version)}
+    with open(TOX_CONFIG, 'w') as configfile:
+        config.write(configfile)
+
+@click.command()
+@click.option('--cassandra', '-c', multiple=True, type=str)
+@click.option('--python', '-p', multiple=True, type=click.Choice(["py2","py3"], case_sensitive = False))
+@click.option("--pytest", "-t", type=str, help="Arguments to be passed to pytest")
+def runtests(cassandra, python, pytest):
+    client = docker.from_env()
+    tox_args = ["-e {}".format(py) for py in python] if python else []
+    if pytest:
+        tox_args.append("--")
+        tox_args.append(pytest)
+    print("Full tox args: {}".format(tox_args))
+
+    cassandra_versions = cassandra or DEFAULT_CASSANDRA_VERSIONS
+    print("Cassandra versions to test: {}".format(','.join(cassandra_versions)))
+    for version in cassandra_versions:
+
+        print("Running test suite for Cassandra version {}".format(version))
+        container = runCassandraContainer(client, version)
+
+        print("Validating connection to local Cassandra")
+        connectToLocalCassandra()
+
+        try:
+            if os.path.exists(TOX_CONFIG):
+                os.remove(TOX_CONFIG)
+            writeToxIni(version)
+
+            # cmdline() will raise SystemExit when it's done so trap that here to avoid
+            # exiting all the things
+            try:
+                tox.cmdline(tox_args)
+            except SystemExit:
+                pass
+        except Exception as exc:
+            print("Exception running tests for Cassandra version {}".format(version), exc)
+        finally:
+            container.stop()
+
+
+if __name__ == '__main__':
+    runtests(obj={})

python/adelphi/test-requirements.txt

@@ -0,0 +1,4 @@
+click ~= 7.1
+cassandra-driver ~= 3.24
+docker ~= 4.4
+tenacity ~= 7.0

python/adelphi/tests/integration/__init__.py

@@ -1,26 +1,45 @@
 import logging
 import os
 import shutil
-import sys
 import tempfile
-import time
 
-from collections import namedtuple
-
-from cassandra.cluster import Cluster
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
-import docker
-from tenacity import retry, stop_after_attempt, wait_fixed
+from collections import namedtuple
 
-# Default C* versions to include in all integration tests
-CASSANDRA_VERSIONS = ["2.1.22", "2.2.19", "3.0.23", "3.11.9", "4.0-beta4"]
+from tests.util.cassandra_util import callWithCassandra, createSchema
 
-logging.basicConfig(filename="adelphi.log", level=logging.INFO)
 log = logging.getLogger('adelphi')
 
 TempDirs = namedtuple('TempDirs', 'basePath, outputDirPath')
 
-class SchemaTestMixin:
+
+def __keyspacesForCluster(cluster):
+    return set(cluster.metadata.keyspaces.keys())
+
+
+def setupSchema(schemaPath):
+    return callWithCassandra(lambda _,s: createSchema(s, schemaPath))
+
+
+def getAllKeyspaces():
+    return callWithCassandra(lambda c,s: __keyspacesForCluster(c))
+
+
+def dropNewKeyspaces(origKeyspaces):
+    def dropFn(cluster, session):
+        currentKeyspaces = __keyspacesForCluster(cluster)
+        droppingKeyspaces = currentKeyspaces - origKeyspaces
+        log.info("Dropping the following keyspaes created by this test: {}".format(",".join(droppingKeyspaces)))
+        for keyspace in droppingKeyspaces:
+            session.execute("drop keyspace {}".format(keyspace))
+    return callWithCassandra(dropFn)
+
+
+class SchemaTestCase(unittest.TestCase):
 
     def basePath(self, name):
         return os.path.join(self.dirs.basePath, name)
@@ -45,96 +64,23 @@ def makeTempDirs(self):
         self.dirs = TempDirs(base, outputDir)
 
 
-    def connectToLocalCassandra(self):
-        session = None
-        while not session:
-            try:
-                cluster = Cluster(["127.0.0.1"],port=9042)
-                session = cluster.connect()
-
-                # Confirm that the session is actually functioning before calling things good
-                rs = session.execute("select * from system.local")
-                log.info("Connected to Cassandra cluster, first row of system.local: {}".format(rs.one()))
-                log.info("Cassandra cluster ready")
-                return (cluster,session)
-            except:
-                log.info("Couldn't quite connect yet, will retry")
-                time.sleep(1)
-
-
-    def createSchema(self, session=None):
-        schemaPath = self.getBaseSchemaPath()
-        log.info("Creating schema on Cassandra cluster from file {}".format(schemaPath))
-        with open(schemaPath) as schema:
-            buff = ""
-            for line in schema:
-                realLine = line.strip()
-                if len(realLine) == 0:
-                    log.debug("Skipping empty statement")
-                    continue
-                if realLine.startswith("//"):
-                    log.debug("Skipping commented statement {}".format(stmt))
-                    continue
-                buff += (" " if len(buff) > 0 else "")
-                buff += realLine
-                if realLine.endswith(';'):
-                    log.debug("Executing statement {}".format(buff))
-                    try:
-                        session.execute(buff)
-                    except:
-                        log.error("Exception executing statement: {}".format(buff), exc_info=sys.exc_info()[0])
-                        self.fail("Exception executing statement: {}, check log for details".format(buff))
-                    buff = ""
-
-
-    @retry(stop=stop_after_attempt(3), wait=wait_fixed(2))
-    def getContainer(self, client, version):
-        return client.containers.run(name="adelphi", remove=True, detach=True, ports={9042:9042}, image="cassandra:{}".format(version))
-
-
-    def runTestForVersion(self, version=None):
-        log.info("Testing Cassandra version {}".format(version))
-
-        client = docker.from_env()
-        container = self.getContainer(client, version)
-
-        self.makeTempDirs()
-
-        (cluster,session) = (None,None)
-        try:
-            (cluster,session) = self.connectToLocalCassandra()
-            self.createSchema(session)
-            log.info("Running Adelphi")
-            self.runAdelphi(version)
-            log.info("Adelphi run completed, evaluating Adelphi output(s)")
-            self.evalAdelphiOutput(version)
-        except:
-            log.error("Exception running test for version {}".format(version), exc_info=sys.exc_info()[0])
-            self.fail("Exception running test for version {}, check log for details".format(version))
-        finally:
-            if cluster:
-                cluster.shutdown()
+    def setUp(self):
+        # Invoking for completeness; for unittest base setUp/tearDown impls are no-ops
+        super(SchemaTestCase, self).setUp()
 
-            if "KEEP_CONTAINER" in os.environ:
-                log.info("KEEP_CONTAINER env var set, preserving Cassandra container 'adelphi'")
-            else:
-                container.stop()
+        # This should be set in the tox config
+        self.version = os.environ["CASSANDRA_VERSION"]
+        log.info("Testing Cassandra version {}".format(self.version))
 
-            self.cleanUpVersion(version)
-
-
-    def testVersions(self):
-        versions = CASSANDRA_VERSIONS
-        if "CASSANDRA_VERSIONS" in os.environ:
-            versions = [s.strip() for s in os.environ["CASSANDRA_VERSIONS"].split(',')]
-
-        log.info("Testing the following Cassandra versions: {}".format(versions))
+        self.makeTempDirs()
 
-        for version in versions:
-            self.runTestForVersion(version)
 
+    def tearDown(self):
+        super(SchemaTestCase, self).tearDown()
 
-    def cleanUpVersion(self, version):
+        # TODO: Note that there's no easy way to access this from test-adelphi unless we modify the
+        # ini generation code... and I'm not completely sure that's worth it.  Might want to think
+        # about just deleting this outright... or making it a CLI option that can be easily accessed.
         if "KEEP_LOGS" in os.environ:
             log.info("KEEP_LOGS env var set, preserving logs/output at {}".format(self.dirs.basePath))
         else: