Implement verdi database migrate

Up till now, the triggering of a database migration happened through
separate interfaces for the two different backends. For Django, if
the database schema did not match the code's, a warning message would
be printed asking the user to execute a stand alone python script that
was a modified Django `manage.py` script. For SqlAlchemy, the user was
automatically prompted with an Alembic migration.

Here we unify the migration operation by making it go through `verdi`
through the endpoint `verdi database migrate`. The error message upon
an outdated database schema and the to be executed commands are now
identical for both backends.

aiida/backends/djsite/__init__.py

@@ -7,5 +7,3 @@
 # For further information on the license, see the LICENSE.txt file        #
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
-
-

aiida/backends/djsite/db/models.py

@@ -554,9 +554,7 @@ class DbMultipleValueAttributeBaseClass(m.Model):
     Abstract base class for tables storing attribute + value data, of
     different data types (without any association to a Node).
     """
-    from aiida.backends.djsite.utils import long_field_length
-
-    key = m.CharField(max_length=long_field_length(), db_index=True, blank=False)
+    key = m.CharField(max_length=1024, db_index=True, blank=False)
     datatype = m.CharField(max_length=10,
                            default='none',
                            choices=attrdatatype_choice, db_index=True)

aiida/backends/djsite/utils.py

@@ -11,10 +11,10 @@
 This modules contains a number of utility functions specific to the
 Django backend.
 """
-
 from __future__ import division
 from __future__ import print_function
 from __future__ import absolute_import
+
 import os
 import django
 
@@ -30,7 +30,7 @@ def load_dbenv(profile=None):
     """
     _load_dbenv_noschemacheck(profile)
     # Check schema version and the existence of the needed tables
-    check_schema_version(profile)
+    check_schema_version(profile_name=profile)
 
 
 def _load_dbenv_noschemacheck(profile):  # pylint: disable=unused-argument
@@ -53,27 +53,13 @@ def _load_dbenv_noschemacheck(profile):  # pylint: disable=unused-argument
 _aiida_autouser_cache = None  # pylint: disable=invalid-name
 
 
-def long_field_length():
-    """
-    Return the length of "long" fields.
-    This is used, for instance, for the 'key' field of attributes.
-    This returns 1024 typically, but it returns 255 if the backend is mysql.
-
-    :note: Call this function only AFTER having called load_dbenv!
-    """
-    # One should not load directly settings because there are checks inside
-    # for the current profile. However, this function is going to be called
-    # only after having loaded load_dbenv, so there should be no problem
-    from django.conf import settings
+def migrate_database():
+    """Migrate the database to the latest schema version."""
+    from django.core.management import call_command
+    call_command('migrate')
 
-    if 'mysql' in settings.DATABASES['default']['ENGINE']:
-        return 255
 
-    # else
-    return 1024
-
-
-def check_schema_version(profile):
+def check_schema_version(profile_name=None):
     """
     Check if the version stored in the database is the same of the version
     of the code.
@@ -89,8 +75,9 @@ def check_schema_version(profile):
     :raise ConfigurationError: if the two schema versions do not match.
       Otherwise, just return.
     """
-    import aiida.backends.djsite.db.models
     from django.db import connection
+
+    import aiida.backends.djsite.db.models
     from aiida.common.exceptions import ConfigurationError
 
     # Do not do anything if the table does not exist yet
@@ -105,19 +92,16 @@ def check_schema_version(profile):
         set_db_schema_version(code_schema_version)
         db_schema_version = get_db_schema_version()
 
-    filepath_utils = os.path.abspath(__file__)
-    filepath_manage = os.path.join(os.path.dirname(filepath_utils), 'manage.py')
-
-    if profile is None:
-        from aiida.common.setup import get_default_profile_name
-        profile = get_default_profile_name()
-
     if code_schema_version != db_schema_version:
-        raise ConfigurationError("The code schema version is {}, but the version stored in the "
-                                 "database (DbSetting table) is {}, stopping.\n"
-                                 "To migrate the database to the current version, run the following commands:"
-                                 "\n  verdi daemon stop\n  python {} --aiida-profile={} migrate".format(
-                                     code_schema_version, db_schema_version, filepath_manage, profile))
+        if profile_name is None:
+            from aiida.manage.manager import get_manager
+            manager = get_manager()
+            profile_name = manager.get_profile().name
+
+        raise ConfigurationError('Database schema version {} is outdated compared to the code schema version {}\n'
+                                 'To migrate the database to the current version, run the following commands:'
+                                 '\n  verdi -p {} daemon stop\n  verdi -p {} database migrate'.format(
+                                     db_schema_version, code_schema_version, profile_name, profile_name))
 
 
 def set_db_schema_version(version):
@@ -157,15 +141,3 @@ def delete_nodes_and_connections_django(pks_to_delete):  # pylint: disable=inval
         models.DbLink.objects.filter(Q(input__in=pks_to_delete) | Q(output__in=pks_to_delete)).delete()
         # now delete nodes
         models.DbNode.objects.filter(pk__in=pks_to_delete).delete()
-
-
-def pass_to_django_manage(argv, profile=None):
-    """
-    Call the corresponding django manage.py command
-    """
-    from aiida.backends.utils import load_dbenv as load_dbenv_, is_dbenv_loaded
-    if not is_dbenv_loaded():
-        load_dbenv_(profile=profile)
-
-    from django.core import management
-    management.execute_from_command_line(argv)

aiida/backends/sqlalchemy/tests/migrations.py

@@ -90,7 +90,7 @@ def test_migrations_forward_backward(self):
 
         """
         from aiida.backends.sqlalchemy.tests.migration_test import versions
-        from aiida.backends.sqlalchemy.utils import check_schema_version
+        from aiida.backends.sqlalchemy.utils import migrate_database
 
         try:
             # Constructing the versions directory
@@ -115,7 +115,7 @@ def test_migrations_forward_backward(self):
                                   "None (no version) since the test setUp "
                                   "method should undo all migrations")
             # Migrate the database to the latest version
-            check_schema_version(force_migration=True, alembic_cfg=alembic_cfg)
+            migrate_database(alembic_cfg=alembic_cfg)
             with sa.engine.begin() as connection:
                 alembic_cfg.attributes['connection'] = connection
                 self.assertEquals(get_db_schema_version(alembic_cfg),

aiida/backends/sqlalchemy/utils.py

@@ -103,7 +103,7 @@ def load_dbenv(profile=None, connection=None):
     """
     _load_dbenv_noschemacheck(profile=profile)
     # Check schema version and the existence of the needed tables
-    check_schema_version()
+    check_schema_version(profile_name=profile)
 
 
 def _load_dbenv_noschemacheck(profile=None, connection=None):
@@ -367,62 +367,53 @@ def get_pg_tc(links_table_name,
                             closure_table_child_field=closure_table_child_field)
 
 
-def check_schema_version(force_migration=False, alembic_cfg=None):
+def migrate_database(alembic_cfg=None):
+    """Migrate the database to the latest schema version.
+
+    :param config: alembic configuration to use, will use default if not provided
     """
-    Check if the version stored in the database is the same of the version
-    of the code.
+    from aiida.backends import sqlalchemy as sa
+
+    if alembic_cfg is None:
+        alembic_cfg = get_alembic_conf()
 
-    :note: if the DbSetting table does not exist, this function does not
-      fail. The reason is to avoid to have problems before running the first
-      migrate call.
+    with sa.engine.begin() as connection:
+        alembic_cfg.attributes['connection'] = connection
+        command.upgrade(alembic_cfg, "head")
 
-    :note: if no version is found, the version is set to the version of the
-      code. This is useful to have the code automatically set the DB version
-      at the first code execution.
 
-    :raise ConfigurationError: if the two schema versions do not match.
-      Otherwise, just return.
+def check_schema_version(profile_name=None):
+    """
+    Check if the version stored in the database is the same of the version of the code.
+
+    :raise ConfigurationError: if the two schema versions do not match
     """
-    import sys
-    from aiida.common.utils import query_yes_no
     from aiida.backends import sqlalchemy as sa
     from aiida.backends.settings import IN_DOC_MODE
+    from aiida.common.exceptions import ConfigurationError
 
-    # Early exit if we compile the documentation since the schema
-    # check is not needed and it creates problems with the sqlalchemy
-    # migrations
+    # Early exit if we compile the documentation since the schema check is not needed and it creates problems
+    # with the sqlalchemy migrations
     if IN_DOC_MODE:
         return
 
-    # If an alembic configuration file is given then use that one.
-    if alembic_cfg is None:
-        alembic_cfg = get_alembic_conf()
+    alembic_cfg = get_alembic_conf()
 
-    # Getting the version of the code and the database
-    # Reusing the existing engine (initialized by AiiDA)
+    # Getting the version of the code and the database, reusing the existing engine (initialized by AiiDA)
     with sa.engine.begin() as connection:
         alembic_cfg.attributes['connection'] = connection
         code_schema_version = get_migration_head(alembic_cfg)
         db_schema_version = get_db_schema_version(alembic_cfg)
 
     if code_schema_version != db_schema_version:
-        if db_schema_version is None:
-            print("It is time to perform your first SQLAlchemy migration.")
-        else:
-            print("The code schema version is {}, but the version stored in "
-                  "the database is {}."
-                  .format(code_schema_version, db_schema_version))
-        if force_migration or query_yes_no("Would you like to migrate to the "
-                                           "latest version?", "yes"):
-            print("Migrating to the last version")
-            # Reusing the existing engine (initialized by AiiDA)
-            with sa.engine.begin() as connection:
-                alembic_cfg.attributes['connection'] = connection
-                command.upgrade(alembic_cfg, "head")
-        else:
-            print("No migration is performed. Exiting since database is out "
-                  "of sync with the code.")
-            sys.exit(1)
+        from aiida.manage.manager import get_manager
+        manager = get_manager()
+        profile_name = manager.get_profile().name
+        raise ConfigurationError('Database schema version {} is outdated compared to the code schema version {}\n'
+                                 'To migrate the database to the current version, run the following commands:'
+                                 '\n  verdi -p {} daemon stop\n  verdi -p {} database migrate'.format(
+                                    code_schema_version, db_schema_version, profile_name, profile_name))
+
 
 
 def get_migration_head(config):

aiida/cmdline/commands/cmd_database.py

@@ -15,6 +15,7 @@
 import click
 
 from aiida.cmdline.commands.cmd_verdi import verdi
+from aiida.cmdline.params import options
 from aiida.cmdline.utils import echo
 
 
@@ -24,6 +25,26 @@ def verdi_database():
     pass
 
 
+@verdi_database.command('migrate')
+@options.FORCE()
+def database_migrate(force):
+    """Migrate the database to the latest schema version."""
+    from aiida.manage.manager import get_manager
+
+    manager = get_manager()
+    profile = manager.get_profile()
+    backend = manager._load_backend(schema_check=False)  # pylint: disable=protected-access
+
+    if not force:
+        echo.echo_warning('Migrating your database might take a while.')
+        echo.echo_warning('Before continuing, make sure the daemon is stopped and you have a backup of your database.')
+        echo.echo_warning('', nl=False)
+        confirm_message = 'Are you really sure you want to migrate the database for profile "{}"?'.format(profile.name)
+        click.confirm(confirm_message, abort=True)
+
+    backend.migrate()
+
+
 @verdi_database.group('integrity')
 def verdi_database_integrity():
     """Various commands that will check the integrity of the database and fix potential issues when asked."""
@@ -46,17 +67,11 @@ def database_integrity(apply_patch):
     command will run an analysis to detect duplicate UUIDs in the node table and solve it by generating new UUIDs. Note
     that it will not delete or merge any nodes.
     """
-    from aiida.backends import settings
-    from aiida.backends.utils import _load_dbenv_noschemacheck
-    from aiida.common.setup import get_default_profile_name
     from aiida.manage.database.integrity import deduplicate_node_uuids
+    from aiida.manage.manager import get_manager
 
-    if settings.AIIDADB_PROFILE is not None:
-        profile = settings.AIIDADB_PROFILE
-    else:
-        profile = get_default_profile_name()
-
-    _load_dbenv_noschemacheck(profile)
+    manager = get_manager()
+    manager._load_backend(schema_check=False)  # pylint: disable=protected-access
 
     try:
         messages = deduplicate_node_uuids(dry_run=not apply_patch)

aiida/common/profile.py

@@ -17,6 +17,8 @@
 from aiida.common import setup
 from aiida.common import exceptions
 
+from .extendeddicts import AttributeDict
+
 __all__ = 'Profile', 'get_current_profile_name', 'get_profile_config'
 
 CONFIG_DIR = setup.AIIDA_CONFIG_FOLDER
@@ -95,7 +97,7 @@ class Profile(object):
 
     def __init__(self, name, config):
         self._name = name
-        self._config = config
+        self._config = AttributeDict(config)
 
         # Currently, whether a profile is a test profile is solely determined by its name starting with 'test_'
         self._test_profile = bool(self.name.startswith('test_'))