Merge branch 'master' into user_fix

CHANGELOG.rst

@@ -45,6 +45,8 @@ Added
 * Add new ``--tail`` flag to the ``st2 run`` / ``st2 action execute`` and ``st2 execution re-run``
   CLI command. When this flag is provided, new execution will automatically be followed and tailed
   after it has been scheduled. (new feature) #3867
+* Now a more user-friendly error message is thrown if a cycle is found inside the Jinja template
+  string (e.g. when parameter / variable references itself). (improvement) #3908
 
 Changed
 ~~~~~~~
@@ -243,6 +245,10 @@ Fixed
 * Fix cancellation of delayed action execution for tasks in workflow. (bug fix)
 * Fix timeout of mistral shutdown in systemd service. The fix is done upstream.
   https://review.openstack.org/#/c/499853/ (bug fix)
+* Fix ``st2ctl clean`` not using database connection information from config.
+  This now uses the new ``st2-cleanup-db`` command. (bug fix) #3659
+
+  Contributed by Nick Maludy (Encore Technologies).
 
 Changed
 ~~~~~~~
@@ -253,6 +259,9 @@ Changed
   ``st2 run`` CLI command. #3670
 
   Contributed by Hiroyasu OHYAMA.
+* Added new command ``st2-cleanup-db`` that drops the current StackStorm MongoDB database. #3659
+
+  Contributed by Nick Maludy (Encore Technologies).
 
 2.4.0 - August 23, 2017
 -----------------------

st2api/tests/unit/controllers/v1/test_executions.py

@@ -459,7 +459,7 @@ def test_post_parameter_render_failed(self):
         post_resp = self._do_post(execution, expect_errors=True)
         self.assertEqual(post_resp.status_int, 400)
         self.assertEqual(post_resp.json['faultstring'],
-                         'Dependecy unsatisfied in ABSENT')
+                         'Dependency unsatisfied in variable "ABSENT"')
 
     def test_post_parameter_validation_explicit_none(self):
         execution = copy.deepcopy(LIVE_ACTION_1)

st2common/bin/st2-cleanup-db

@@ -0,0 +1,22 @@
+#!/usr/bin/env python2.7
+# Licensed to the StackStorm, Inc ('StackStorm') under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import sys
+import st2common.persistence.cleanup as db_cleanup
+
+if __name__ == '__main__':
+    sys.exit(db_cleanup.main(sys.argv[1:]))

st2common/bin/st2ctl

@@ -163,7 +163,7 @@ function register_content() {
 
 function clean_db() {
   echo "Dropping st2 Database..."
-  mongo st2 --eval "db.dropDatabase();"
+  /opt/stackstorm/st2/bin/st2-cleanup-db --config-file ${STANCONF}
 }
 
 function clean_logs() {

st2common/setup.py

@@ -46,6 +46,7 @@
     packages=find_packages(exclude=['setuptools', 'tests']),
     scripts=[
         'bin/st2-bootstrap-rmq',
+        'bin/st2-cleanup-db',
         'bin/st2-register-content',
         'bin/st2-apply-rbac-definitions',
         'bin/st2-purge-executions',

st2common/st2common/database_setup.py

@@ -23,26 +23,41 @@
 from st2common.persistence import db_init
 
 __all__ = [
+    'db_config',
     'db_setup',
     'db_teardown'
 ]
 
 
-def db_setup(ensure_indexes=True):
+def db_config():
     username = getattr(cfg.CONF.database, 'username', None)
     password = getattr(cfg.CONF.database, 'password', None)
 
-    connection = db_init.db_setup_with_retry(
-        db_name=cfg.CONF.database.db_name, db_host=cfg.CONF.database.host,
-        db_port=cfg.CONF.database.port, username=username, password=password,
-        ensure_indexes=ensure_indexes,
-        ssl=cfg.CONF.database.ssl, ssl_keyfile=cfg.CONF.database.ssl_keyfile,
-        ssl_certfile=cfg.CONF.database.ssl_certfile,
-        ssl_cert_reqs=cfg.CONF.database.ssl_cert_reqs,
-        ssl_ca_certs=cfg.CONF.database.ssl_ca_certs,
-        ssl_match_hostname=cfg.CONF.database.ssl_match_hostname)
+    return {'db_name': cfg.CONF.database.db_name,
+            'db_host': cfg.CONF.database.host,
+            'db_port': cfg.CONF.database.port,
+            'username': username,
+            'password': password,
+            'ssl': cfg.CONF.database.ssl,
+            'ssl_keyfile': cfg.CONF.database.ssl_keyfile,
+            'ssl_certfile': cfg.CONF.database.ssl_certfile,
+            'ssl_cert_reqs': cfg.CONF.database.ssl_cert_reqs,
+            'ssl_ca_certs': cfg.CONF.database.ssl_ca_certs,
+            'ssl_match_hostname': cfg.CONF.database.ssl_match_hostname}
+
+
+def db_setup(ensure_indexes=True):
+    """
+    Creates the database and indexes (optional).
+    """
+    db_cfg = db_config()
+    db_cfg['ensure_indexes'] = ensure_indexes
+    connection = db_init.db_setup_with_retry(**db_cfg)
     return connection
 
 
 def db_teardown():
+    """
+    Disconnects from the database.
+    """
     return db.db_teardown()

st2common/st2common/models/db/__init__.py

@@ -75,7 +75,7 @@ def get_model_classes():
     return result
 
 
-def db_setup(db_name, db_host, db_port, username=None, password=None, ensure_indexes=True,
+def _db_connect(db_name, db_host, db_port, username=None, password=None,
              ssl=False, ssl_keyfile=None, ssl_certfile=None,
              ssl_cert_reqs=None, ssl_ca_certs=None, ssl_match_hostname=True):
 
@@ -110,6 +110,18 @@ def db_setup(db_name, db_host, db_port, username=None, password=None, ensure_ind
                                                 port=db_port, tz_aware=True,
                                                 username=username, password=password,
                                                 **ssl_kwargs)
+    return connection
+
+
+def db_setup(db_name, db_host, db_port, username=None, password=None, ensure_indexes=True,
+             ssl=False, ssl_keyfile=None, ssl_certfile=None,
+             ssl_cert_reqs=None, ssl_ca_certs=None, ssl_match_hostname=True):
+
+    connection = _db_connect(db_name, db_host, db_port, username=username,
+                             password=password, ssl=ssl, ssl_keyfile=ssl_keyfile,
+                             ssl_certfile=ssl_certfile,
+                             ssl_cert_reqs=ssl_cert_reqs, ssl_ca_certs=ssl_ca_certs,
+                             ssl_match_hostname=ssl_match_hostname)
 
     # Create all the indexes upfront to prevent race-conditions caused by
     # lazy index creation
@@ -222,6 +234,23 @@ def db_teardown():
     mongoengine.connection.disconnect()
 
 
+def db_cleanup(db_name, db_host, db_port, username=None, password=None,
+               ssl=False, ssl_keyfile=None, ssl_certfile=None,
+               ssl_cert_reqs=None, ssl_ca_certs=None, ssl_match_hostname=True):
+
+    connection = _db_connect(db_name, db_host, db_port, username=username,
+                             password=password, ssl=ssl, ssl_keyfile=ssl_keyfile,
+                             ssl_certfile=ssl_certfile,
+                             ssl_cert_reqs=ssl_cert_reqs, ssl_ca_certs=ssl_ca_certs,
+                             ssl_match_hostname=ssl_match_hostname)
+
+    LOG.info('Dropping database "%s" @ "%s:%s" as user "%s".',
+             db_name, db_host, db_port, str(username))
+
+    connection.drop_database(db_name)
+    return connection
+
+
 def _get_ssl_kwargs(ssl=False, ssl_keyfile=None, ssl_certfile=None, ssl_cert_reqs=None,
                     ssl_ca_certs=None, ssl_match_hostname=True):
     ssl_kwargs = {

st2common/st2common/persistence/cleanup.py

@@ -0,0 +1,77 @@
+# Licensed to the StackStorm, Inc ('StackStorm') under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+from st2common import config
+from st2common import log as logging
+from st2common.database_setup import db_config
+from st2common.models.db import db_cleanup as db_cleanup_func
+from st2common.persistence.db_init import db_func_with_retry
+from st2common.script_setup import setup as common_setup
+from st2common.script_setup import teardown as common_teardown
+
+__all__ = [
+    'db_cleanup',
+    'db_cleanup_with_retry',
+    'main'
+]
+
+LOG = logging.getLogger(__name__)
+
+
+def db_cleanup():
+    """
+    Drops the database.
+    """
+    db_cfg = db_config()
+    connection = db_cleanup_with_retry(**db_cfg)
+    return connection
+
+
+def db_cleanup_with_retry(db_name, db_host, db_port, username=None, password=None,
+                          ssl=False, ssl_keyfile=None,
+                          ssl_certfile=None, ssl_cert_reqs=None, ssl_ca_certs=None,
+                          ssl_match_hostname=True):
+    """
+    This method is a retry version of db_cleanup.
+    """
+    return db_func_with_retry(db_cleanup_func,
+                              db_name, db_host, db_port,
+                              username=username, password=password,
+                              ssl=ssl, ssl_keyfile=ssl_keyfile,
+                              ssl_certfile=ssl_certfile, ssl_cert_reqs=ssl_cert_reqs,
+                              ssl_ca_certs=ssl_ca_certs,
+                              ssl_match_hostname=ssl_match_hostname)
+
+
+def setup(argv):
+    common_setup(config=config, setup_db=False, register_mq_exchanges=False,
+                 register_internal_trigger_types=False)
+
+
+def teardown():
+    common_teardown()
+
+
+def main(argv):
+    setup(argv)
+    db_cleanup()
+    teardown()
+
+
+# This script registers actions and rules from content-packs.
+if __name__ == '__main__':
+    main(sys.argv[1:])

st2common/st2common/persistence/db_init.py

@@ -41,12 +41,9 @@ def _retry_if_connection_error(error):
     return is_connection_error
 
 
-def db_setup_with_retry(db_name, db_host, db_port, username=None, password=None,
-                        ensure_indexes=True, ssl=False, ssl_keyfile=None,
-                        ssl_certfile=None, ssl_cert_reqs=None, ssl_ca_certs=None,
-                        ssl_match_hostname=True):
+def db_func_with_retry(db_func, *args, **kwargs):
     """
-    This method is a retry version of db_setup.
+    This method is a generic retry function to support database setup and cleanup.
     """
     # Using as an annotation would be nice but annotations are evaluated at import
     # time and simple ways to use the annotation means the config gets read before
@@ -58,8 +55,20 @@ def db_setup_with_retry(db_name, db_host, db_port, username=None, password=None,
         wait_exponential_max=cfg.CONF.database.connection_retry_backoff_max_s * 1000,
         stop_max_delay=cfg.CONF.database.connection_retry_max_delay_m * 60 * 1000
     )
-    return retrying_obj.call(db_setup, db_name, db_host, db_port, username=username,
-                             password=password, ensure_indexes=ensure_indexes,
-                             ssl=ssl, ssl_keyfile=ssl_keyfile, ssl_certfile=ssl_certfile,
-                             ssl_cert_reqs=ssl_cert_reqs, ssl_ca_certs=ssl_ca_certs,
-                             ssl_match_hostname=ssl_match_hostname)
+    return retrying_obj.call(db_func, *args, **kwargs)
+
+
+def db_setup_with_retry(db_name, db_host, db_port, username=None, password=None,
+                        ensure_indexes=True, ssl=False, ssl_keyfile=None,
+                        ssl_certfile=None, ssl_cert_reqs=None, ssl_ca_certs=None,
+                        ssl_match_hostname=True):
+    """
+    This method is a retry version of db_setup.
+    """
+    return db_func_with_retry(db_setup, db_name, db_host, db_port,
+                              username=username, password=password,
+                              ensure_indexes=ensure_indexes,
+                              ssl=ssl, ssl_keyfile=ssl_keyfile,
+                              ssl_certfile=ssl_certfile, ssl_cert_reqs=ssl_cert_reqs,
+                              ssl_ca_certs=ssl_ca_certs,
+                              ssl_match_hostname=ssl_match_hostname)

st2common/st2common/util/param.py

@@ -141,11 +141,24 @@ def _validate(G):
     '''
     for name in G.nodes():
         if 'value' not in G.node[name] and 'template' not in G.node[name]:
-            msg = 'Dependecy unsatisfied in %s' % name
+            msg = 'Dependency unsatisfied in variable "%s"' % name
             raise ParamException(msg)
 
     if not nx.is_directed_acyclic_graph(G):
-        msg = 'Cyclic dependecy found'
+        graph_cycles = nx.simple_cycles(G)
+
+        variable_names = []
+        for cycle in graph_cycles:
+            try:
+                variable_name = cycle[0]
+            except IndexError:
+                continue
+
+            variable_names.append(variable_name)
+
+        variable_names = ', '.join(variable_names)
+        msg = ('Cyclic dependency found in the following variables: %s. Likely the variable is '
+               'referencing itself' % (variable_names))
         raise ParamException(msg)
 
 
@@ -159,7 +172,7 @@ def _render(node, render_context):
         if isinstance(node['template'], list) or isinstance(node['template'], dict):
             node['template'] = json.dumps(node['template'])
 
-            # Finds occourances of "{{variable}}" and adds `to_complex` filter
+            # Finds occurrences of "{{variable}}" and adds `to_complex` filter
             # so types are honored. If it doesn't follow that syntax then it's
             # rendered as a string.
             node['template'] = re.sub(

st2common/tests/unit/test_db.py

@@ -28,6 +28,7 @@
 from st2common.exceptions.db import StackStormDBObjectNotFoundError
 from st2common.models.db.trigger import TriggerTypeDB, TriggerDB, TriggerInstanceDB
 from st2common.models.db.rule import RuleDB, ActionExecutionSpecDB
+from st2common.persistence.cleanup import db_cleanup
 from st2common.persistence.rule import Rule
 from st2common.persistence.trigger import TriggerType, Trigger, TriggerInstance
 from st2tests import DbTestCase
@@ -115,6 +116,19 @@ def test_db_setup_connecting_info_logging(self, mock_log):
         self.assertEqual(expected_message, actual_message)
 
 
+class DbCleanupTest(DbTestCase):
+
+    def test_cleanup(self):
+        """
+        Tests dropping the database. Requires the db server to be running.
+        """
+        self.assertIn(cfg.CONF.database.db_name, self.db_connection.database_names())
+
+        connection = db_cleanup()
+
+        self.assertNotIn(cfg.CONF.database.db_name, connection.database_names())
+
+
 @mock.patch.object(PoolPublisher, 'publish', mock.MagicMock())
 class ReactorModelTest(DbTestCase):