Skip to content

Commit

Permalink
Replace verdi database commands with verdi storage (#5228)
Browse files Browse the repository at this point in the history 
The following specific commands were moved:

 - integrity: the checks that were performed here are no longer relevant.
   There will be a new verdi storage integrity that will perform more
   general checks (once the features of the repository are implemented).

 - migrate: moved to `verdi storage migrate`.

 - summary: information now available through `verdi storage info`.

 - version: information now available through `verdi status`.

The respective tests were adapted as well.

Co-authored-by: Sebastiaan Huber <mail@sphuber.net>
  • Loading branch information
@ramirezfranciscof @sphuber
ramirezfranciscof and sphuber authored Nov 16, 2021
1 parent 370358d commit f8d1615
Show file tree
Hide file tree
Showing 8 changed files with 349 additions and 367 deletions.
1 change: 1 addition & 0 deletions aiida/cmdline/commands/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,5 +32,6 @@
cmd_setup,
cmd_shell,
cmd_status,
cmd_storage,
cmd_user,
)
221 changes: 56 additions & 165 deletions aiida/cmdline/commands/cmd_database.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,95 +8,60 @@
# For further information please visit http://www.aiida.net #
###########################################################################
"""`verdi database` commands."""
# pylint: disable=unused-argument

import click

from aiida.backends.general.migrations.duplicate_uuids import TABLES_UUID_DEDUPLICATION
from aiida.cmdline.commands.cmd_verdi import verdi
from aiida.cmdline.params import options
from aiida.cmdline.utils import decorators, echo
from aiida.common import exceptions
from aiida.cmdline.utils import decorators


@verdi.group('database')
def verdi_database():
"""Inspect and manage the database."""
"""Inspect and manage the database.
.. deprecated:: v2.0.0
"""


@verdi_database.command('version')
@decorators.deprecated_command(
'This command has been deprecated and no longer has any effect. It will be removed soon from the CLI (in v2.1).\n'
'The same information is now available through `verdi status`.\n'
)
def database_version():
"""Show the version of the database.
The database version is defined by the tuple of the schema generation and schema revision.
"""
from aiida.manage.manager import get_manager

manager = get_manager()
manager._load_backend(schema_check=False) # pylint: disable=protected-access
backend_manager = manager.get_backend_manager()
echo.echo('Generation: ', bold=True, nl=False)
echo.echo(backend_manager.get_schema_generation_database())
echo.echo('Revision: ', bold=True, nl=False)
echo.echo(backend_manager.get_schema_version_backend())
.. deprecated:: v2.0.0
"""


@verdi_database.command('migrate')
@options.FORCE()
def database_migrate(force):
"""Migrate the database to the latest schema version."""
from aiida.engine.daemon.client import get_daemon_client
from aiida.manage.manager import get_manager

client = get_daemon_client()
if client.is_daemon_running:
echo.echo_critical('Migration aborted, the daemon for the profile is still running.')

manager = get_manager()
profile = manager.get_profile()
backend = manager._load_backend(schema_check=False) # pylint: disable=protected-access

if force:
try:
backend.migrate()
except (exceptions.ConfigurationError, exceptions.DatabaseMigrationError) as exception:
echo.echo_critical(str(exception))
return

echo.echo_warning('Migrating your database might take a while and is not reversible.')
echo.echo_warning('Before continuing, make sure you have completed the following steps:')
echo.echo_warning('')
echo.echo_warning(' 1. Make sure you have no active calculations and workflows.')
echo.echo_warning(' 2. If you do, revert the code to the previous version and finish running them first.')
echo.echo_warning(' 3. Stop the daemon using `verdi daemon stop`')
echo.echo_warning(' 4. Make a backup of your database and repository')
echo.echo_warning('')
echo.echo_warning('', nl=False)

expected_answer = 'MIGRATE NOW'
confirm_message = 'If you have completed the steps above and want to migrate profile "{}", type {}'.format(
profile.name, expected_answer
)

try:
response = click.prompt(confirm_message)
while response != expected_answer:
response = click.prompt(confirm_message)
except click.Abort:
echo.echo('\n')
echo.echo_critical('Migration aborted, the data has not been affected.')
else:
try:
backend.migrate()
except (exceptions.ConfigurationError, exceptions.DatabaseMigrationError) as exception:
echo.echo_critical(str(exception))
else:
echo.echo_success('migration completed')
@click.pass_context
@decorators.deprecated_command(
'This command has been deprecated and will be removed soon (in v3.0). '
'Please call `verdi storage migrate` instead.\n'
)
def database_migrate(ctx, force):
"""Migrate the database to the latest schema version.
.. deprecated:: v2.0.0
"""
from aiida.cmdline.commands.cmd_storage import backend_migrate
ctx.forward(backend_migrate)


@verdi_database.group('integrity')
def verdi_database_integrity():
"""Check the integrity of the database and fix potential issues."""
"""Check the integrity of the database and fix potential issues.
.. deprecated:: v2.0.0
"""


@verdi_database_integrity.command('detect-duplicate-uuid')
Expand All @@ -110,6 +75,10 @@ def verdi_database_integrity():
@click.option(
'-a', '--apply-patch', is_flag=True, help='Actually apply the proposed changes instead of performing a dry run.'
)
@decorators.deprecated_command(
'This command has been deprecated and no longer has any effect. It will be removed soon from the CLI (in v2.1).\n'
'For remaining available integrity checks, use `verdi storage integrity` instead.\n'
)
def detect_duplicate_uuid(table, apply_patch):
"""Detect and fix entities with duplicate UUIDs.
Expand All @@ -119,123 +88,45 @@ def detect_duplicate_uuid(table, apply_patch):
constraint on UUIDs on the database level. However, this would leave databases created before this patch with
duplicate UUIDs in an inconsistent state. This command will run an analysis to detect duplicate UUIDs in a given
table and solve it by generating new UUIDs. Note that it will not delete or merge any rows.
"""
from aiida.backends.general.migrations.duplicate_uuids import deduplicate_uuids
from aiida.manage.manager import get_manager
manager = get_manager()
manager._load_backend(schema_check=False) # pylint: disable=protected-access
try:
messages = deduplicate_uuids(table=table, dry_run=not apply_patch)
except Exception as exception: # pylint: disable=broad-except
echo.echo_critical(f'integrity check failed: {str(exception)}')
else:
for message in messages:
echo.echo_report(message)

if apply_patch:
echo.echo_success('integrity patch completed')
else:
echo.echo_success('dry-run of integrity patch completed')
.. deprecated:: v2.0.0
"""


@verdi_database_integrity.command('detect-invalid-links')
@decorators.with_dbenv()
@decorators.deprecated_command(
'This command has been deprecated and no longer has any effect. It will be removed soon from the CLI (in v2.1).\n'
'For remaining available integrity checks, use `verdi storage integrity` instead.\n'
)
def detect_invalid_links():
"""Scan the database for invalid links."""
from tabulate import tabulate

from aiida.manage.database.integrity.sql.links import INVALID_LINK_SELECT_STATEMENTS
from aiida.manage.manager import get_manager
"""Scan the database for invalid links.
integrity_violated = False

backend = get_manager().get_backend()

for check in INVALID_LINK_SELECT_STATEMENTS:

result = backend.execute_prepared_statement(check.sql, check.parameters)

if result:
integrity_violated = True
echo.echo_warning(f'{check.message}:\n')
echo.echo(tabulate(result, headers=check.headers))

if not integrity_violated:
echo.echo_success('no integrity violations detected')
else:
echo.echo_critical('one or more integrity violations detected')
.. deprecated:: v2.0.0
"""


@verdi_database_integrity.command('detect-invalid-nodes')
@decorators.with_dbenv()
@decorators.deprecated_command(
'This command has been deprecated and no longer has any effect. It will be removed soon from the CLI (in v2.1).\n'
'For remaining available integrity checks, use `verdi storage integrity` instead.\n'
)
def detect_invalid_nodes():
"""Scan the database for invalid nodes."""
from tabulate import tabulate

from aiida.manage.database.integrity.sql.nodes import INVALID_NODE_SELECT_STATEMENTS
from aiida.manage.manager import get_manager

integrity_violated = False
"""Scan the database for invalid nodes.
backend = get_manager().get_backend()

for check in INVALID_NODE_SELECT_STATEMENTS:

result = backend.execute_prepared_statement(check.sql, check.parameters)

if result:
integrity_violated = True
echo.echo_warning(f'{check.message}:\n')
echo.echo(tabulate(result, headers=check.headers))

if not integrity_violated:
echo.echo_success('no integrity violations detected')
else:
echo.echo_critical('one or more integrity violations detected')
.. deprecated:: v2.0.0
"""


@verdi_database.command('summary')
@decorators.deprecated_command(
'This command has been deprecated and no longer has any effect. It will be removed soon from the CLI (in v2.1).\n'
'Please call `verdi storage info` instead.\n'
)
def database_summary():
"""Summarise the entities in the database."""
from aiida.cmdline import is_verbose
from aiida.orm import Comment, Computer, Group, Log, Node, QueryBuilder, User
data = {}

# User
query_user = QueryBuilder().append(User, project=['email'])
data['Users'] = {'count': query_user.count()}
if is_verbose():
data['Users']['emails'] = query_user.distinct().all(flat=True)

# Computer
query_comp = QueryBuilder().append(Computer, project=['label'])
data['Computers'] = {'count': query_comp.count()}
if is_verbose():
data['Computers']['labels'] = query_comp.distinct().all(flat=True)

# Node
count = QueryBuilder().append(Node).count()
data['Nodes'] = {'count': count}
if is_verbose():
node_types = QueryBuilder().append(Node, project=['node_type']).distinct().all(flat=True)
data['Nodes']['node_types'] = node_types
process_types = QueryBuilder().append(Node, project=['process_type']).distinct().all(flat=True)
data['Nodes']['process_types'] = [p for p in process_types if p]

# Group
query_group = QueryBuilder().append(Group, project=['type_string'])
data['Groups'] = {'count': query_group.count()}
if is_verbose():
data['Groups']['type_strings'] = query_group.distinct().all(flat=True)

# Comment
count = QueryBuilder().append(Comment).count()
data['Comments'] = {'count': count}

# Log
count = QueryBuilder().append(Log).count()
data['Logs'] = {'count': count}

echo.echo_dictionary(data, sort_keys=False, fmt='yaml')
"""Summarise the entities in the database.
.. deprecated:: v2.0.0
"""
37 changes: 26 additions & 11 deletions aiida/cmdline/commands/cmd_status.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ class ServiceStatus(enum.IntEnum):
@click.option('--no-rmq', is_flag=True, help='Do not check RabbitMQ status')
def verdi_status(print_traceback, no_rmq):
"""Print status of AiiDA services."""
# pylint: disable=broad-except,too-many-statements,too-many-branches
# pylint: disable=broad-except,too-many-statements,too-many-branches,too-many-locals,
from aiida import __version__
from aiida.cmdline.utils.daemon import delete_stale_pid_file, get_daemon_status
from aiida.common.utils import Capturing
Expand All @@ -68,16 +68,17 @@ def verdi_status(print_traceback, no_rmq):
print_status(ServiceStatus.UP, 'config', AIIDA_CONFIG_FOLDER)

manager = get_manager()
profile = manager.get_profile()

if profile is None:
print_status(ServiceStatus.WARNING, 'profile', 'no profile configured yet')
echo.echo_report('Configure a profile by running `verdi quicksetup` or `verdi setup`.')
return

try:
profile = manager.get_profile()

if profile is None:
print_status(ServiceStatus.WARNING, 'profile', 'no profile configured yet')
echo.echo_report('Configure a profile by running `verdi quicksetup` or `verdi setup`.')
return

print_status(ServiceStatus.UP, 'profile', profile.name)

except Exception as exc:
message = 'Unable to read AiiDA profile'
print_status(ServiceStatus.ERROR, 'profile', message, exception=exc, print_traceback=print_traceback)
Expand All @@ -95,21 +96,35 @@ def verdi_status(print_traceback, no_rmq):
print_status(ServiceStatus.UP, 'repository', repository_status)

# Getting the postgres status by trying to get a database cursor
database_data = [profile.database_name, profile.database_username, profile.database_hostname, profile.database_port]
backend_manager = manager.get_backend_manager()
dbgen = backend_manager.get_schema_generation_database()
dbver = backend_manager.get_schema_version_backend()
database_data = [
profile.database_name, dbgen, dbver, profile.database_username, profile.database_hostname, profile.database_port
]
try:
with override_log_level(): # temporarily suppress noisy logging
backend = manager.get_backend()
backend.cursor()

except IncompatibleDatabaseSchema:
message = 'Database schema version is incompatible with the code: run `verdi database migrate`.'
message = f'Database schema {dbgen} / {dbver} (generation/version) is incompatible with the code. '
message += 'Run `verdi database migrate` to solve this.'
print_status(ServiceStatus.DOWN, 'postgres', message)
exit_code = ExitCode.CRITICAL

except Exception as exc:
message = 'Unable to connect to database `{}` as {}@{}:{}'.format(*database_data)
message = 'Unable to connect to database `{}` with schema {} / {} (generation/version) as {}@{}:{}'.format(
*database_data
)
print_status(ServiceStatus.DOWN, 'postgres', message, exception=exc, print_traceback=print_traceback)
exit_code = ExitCode.CRITICAL

else:
print_status(ServiceStatus.UP, 'postgres', 'Connected to database `{}` as {}@{}:{}'.format(*database_data))
message = 'Connected to database `{}` with schema {} / {} (generation/version) as {}@{}:{}'.format(
*database_data
)
print_status(ServiceStatus.UP, 'postgres', message)

# Getting the rmq status
if not no_rmq:
Expand Down
Loading

0 comments on commit f8d1615

Please sign in to comment.