🔀 MERGE: Refactor/rename storage modules (#5364)

This is a follow-up to #5330,
which separates the storage backend abstraction and implementations,
with the abstraction in `aiida.orm`
and implementations moving to `aiida/storage`.

.pre-commit-config.yaml

@@ -70,25 +70,21 @@ repos:
         pass_filenames: true
         files: >-
             (?x)^(
-                aiida/backends/control.py|
                 aiida/common/progress_reporter.py|
                 aiida/engine/.*py|
                 aiida/manage/manager.py|
                 aiida/manage/database/delete/nodes.py|
                 aiida/orm/querybuilder.py|
                 aiida/orm/implementation/entities.py|
-                aiida/orm/implementation/backends.py|
+                aiida/orm/implementation/storage_backend.py|
                 aiida/orm/implementation/authinfos.py|
                 aiida/orm/implementation/comments.py|
                 aiida/orm/implementation/computers.py|
                 aiida/orm/implementation/groups.py|
                 aiida/orm/implementation/logs.py|
                 aiida/orm/implementation/nodes.py|
                 aiida/orm/implementation/users.py|
-                aiida/orm/implementation/sql/backends.py|
-                aiida/orm/implementation/sqlalchemy/backend.py|
                 aiida/orm/implementation/querybuilder.py|
-                aiida/orm/implementation/sqlalchemy/querybuilder/.*py|
                 aiida/orm/entities.py|
                 aiida/orm/authinfos.py|
                 aiida/orm/comments.py|
@@ -103,6 +99,10 @@ repos:
                 aiida/plugins/entry_point.py|
                 aiida/plugins/factories.py|
                 aiida/repository/.*py|
+                aiida/storage/control.py|
+                aiida/storage/psql_dos/backend.py|
+                aiida/storage/psql_dos/orm/querybuilder/.*py|
+                aiida/storage/psql_dos/utils.py|
                 aiida/tools/graph/graph_traversers.py|
                 aiida/tools/groups/paths.py|
                 aiida/tools/archive/.*py|

CHANGELOG.md

@@ -702,7 +702,7 @@ Changes between 1.0 alpha/beta releases are not included - for those see the cha
 - AiiDA now enforces UTF-8 encoding for text output in its files and databases. [[#2107]](https://github.com/aiidateam/aiida-core/pull/2107)
 
 #### Backwards-incompatible changes (only a sub-set)
-- Remove `aiida.tests` and obsolete `aiida.backends.tests.test_parsers` entry point group [[#2778]](https://github.com/aiidateam/aiida-core/pull/2778)
+- Remove `aiida.tests` and obsolete `aiida.storage.tests.test_parsers` entry point group [[#2778]](https://github.com/aiidateam/aiida-core/pull/2778)
 - Implement new link types [[#2220]](https://github.com/aiidateam/aiida-core/pull/2220)
 - Rename the type strings of `Groups` and change the attributes `name` and `type` to `label` and `type_string` [[#2329]](https://github.com/aiidateam/aiida-core/pull/2329)
 - Make various protected `Node` methods public [[#2544]](https://github.com/aiidateam/aiida-core/pull/2544)

aiida/cmdline/commands/cmd_devel.py

@@ -45,7 +45,7 @@ def devel_check_load_time():
     if manager.profile_storage_loaded:
         echo.echo_critical('potential `verdi` speed problem: database backend is loaded.')
 
-    allowed = ('aiida.backends', 'aiida.cmdline', 'aiida.common', 'aiida.manage', 'aiida.plugins', 'aiida.restapi')
+    allowed = ('aiida.cmdline', 'aiida.common', 'aiida.manage', 'aiida.plugins', 'aiida.restapi')
     for loaded in loaded_aiida_modules:
         if not any(loaded.startswith(mod) for mod in allowed):
             echo.echo_critical(
@@ -102,7 +102,7 @@ def devel_run_sql(sql):
     """Run a raw SQL command on the profile database (only available for 'psql_dos' storage)."""
     from sqlalchemy import text
 
-    from aiida.backends.sqlalchemy.utils import create_sqlalchemy_engine
+    from aiida.storage.psql_dos.utils import create_sqlalchemy_engine
     assert get_profile().storage_backend == 'psql_dos'
     with create_sqlalchemy_engine(get_profile().storage_config).connect() as connection:
         result = connection.execute(text(sql)).fetchall()

aiida/cmdline/commands/cmd_storage.py

@@ -92,9 +92,9 @@ def storage_integrity():
 @click.option('--statistics', is_flag=True, help='Provides more in-detail statistically relevant data.')
 def storage_info(statistics):
     """Summarise the contents of the storage."""
-    from aiida.backends.control import get_repository_info
     from aiida.cmdline.utils.common import get_database_summary
     from aiida.orm import QueryBuilder
+    from aiida.storage.control import get_repository_info
 
     with spinner():
         data = {
@@ -119,7 +119,7 @@ def storage_info(statistics):
 )
 def storage_maintain(full, dry_run):
     """Performs maintenance tasks on the repository."""
-    from aiida.backends.control import repository_maintain
+    from aiida.storage.control import repository_maintain
 
     if full:
         echo.echo_warning(

aiida/common/utils.py

@@ -24,8 +24,6 @@
 def get_new_uuid():
     """
     Return a new UUID (typically to be used for new nodes).
-    It uses the UUID version specified in
-    aiida.backends.settings.AIIDANODES_UUID_VERSION
     """
     import uuid
     return str(uuid.uuid4())

aiida/manage/configuration/__init__.py

@@ -248,7 +248,7 @@ def load_documentation_profile():
     """
     import tempfile
 
-    from aiida.backends.sqlalchemy.models.base import get_orm_metadata
+    from aiida.storage.psql_dos.models.base import get_orm_metadata
 
     from .config import Config
 

aiida/manage/configuration/profile.py

@@ -20,7 +20,7 @@
 from .settings import DAEMON_DIR, DAEMON_LOG_DIR
 
 if TYPE_CHECKING:
-    from aiida.orm.implementation.backends import Backend
+    from aiida.orm.implementation import StorageBackend
 
 __all__ = ('Profile',)
 
@@ -121,10 +121,10 @@ def set_storage(self, name: str, config: Dict[str, Any]) -> None:
         self._attributes[self.KEY_STORAGE][self.KEY_STORAGE_CONFIG] = config
 
     @property
-    def storage_cls(self) -> Type['Backend']:
+    def storage_cls(self) -> Type['StorageBackend']:
         """Return the storage backend class for this profile."""
         if self.storage_backend == 'psql_dos':
-            from aiida.orm.implementation.sqlalchemy.backend import PsqlDosBackend
+            from aiida.storage.psql_dos.backend import PsqlDosBackend
             return PsqlDosBackend
         if self.storage_backend == 'archive.sqlite':
             from aiida.tools.archive.implementations.sqlite.backend import ArchiveReadOnlyBackend

aiida/manage/manager.py

@@ -23,7 +23,7 @@
     from aiida.engine.runners import Runner
     from aiida.manage.configuration.config import Config
     from aiida.manage.configuration.profile import Profile
-    from aiida.orm.implementation import Backend
+    from aiida.orm.implementation import StorageBackend
 
 __all__ = ('get_manager',)
 
@@ -64,7 +64,7 @@ class Manager:
     def __init__(self) -> None:
         # note: the config currently references the global variables
         self._profile: Optional['Profile'] = None
-        self._profile_storage: Optional['Backend'] = None
+        self._profile_storage: Optional['StorageBackend'] = None
         self._daemon_client: Optional['DaemonClient'] = None
         self._communicator: Optional['RmqThreadCommunicator'] = None
         self._process_controller: Optional['RemoteProcessThreadController'] = None
@@ -193,7 +193,7 @@ def get_option(self, option_name: str) -> Any:
         option = get_option(option_name)
         return option.default
 
-    def get_backend(self) -> 'Backend':
+    def get_backend(self) -> 'StorageBackend':
         """Return the current profile's storage backend, loading it if necessary.
 
         Deprecated: use `get_profile_storage` instead.
@@ -202,7 +202,7 @@ def get_backend(self) -> 'Backend':
         warn('get_backend() is deprecated, use get_profile_storage() instead', AiidaDeprecationWarning)
         return self.get_profile_storage()
 
-    def get_profile_storage(self) -> 'Backend':
+    def get_profile_storage(self) -> 'StorageBackend':
         """Return the current profile's storage backend, loading it if necessary."""
         from aiida.common import ConfigurationError
         from aiida.common.log import configure_logging

aiida/manage/tests/main.py

@@ -147,7 +147,7 @@ def __init__(self, profile_name):
         :param profile_name: Name of the profile to be loaded
         """
         from aiida import load_profile
-        from aiida.backends.testbase import check_if_tests_can_run
+        from aiida.storage.testbase import check_if_tests_can_run
 
         self._profile = None
         try:

aiida/orm/authinfos.py

@@ -19,7 +19,7 @@
 
 if TYPE_CHECKING:
     from aiida.orm import Computer, User
-    from aiida.orm.implementation import Backend, BackendAuthInfo
+    from aiida.orm.implementation import BackendAuthInfo, StorageBackend
     from aiida.transports import Transport
 
 __all__ = ('AuthInfo',)
@@ -51,7 +51,7 @@ def objects(cls: Type['AuthInfo']) -> AuthInfoCollection:  # type: ignore[misc]
 
     PROPERTY_WORKDIR = 'workdir'
 
-    def __init__(self, computer: 'Computer', user: 'User', backend: Optional['Backend'] = None) -> None:
+    def __init__(self, computer: 'Computer', user: 'User', backend: Optional['StorageBackend'] = None) -> None:
         """Create an `AuthInfo` instance for the given computer and user.
 
         :param computer: a `Computer` instance

aiida/orm/comments.py

@@ -18,7 +18,7 @@
 
 if TYPE_CHECKING:
     from aiida.orm import Node, User
-    from aiida.orm.implementation import Backend, BackendComment
+    from aiida.orm.implementation import BackendComment, StorageBackend
 
 __all__ = ('Comment',)
 
@@ -72,7 +72,9 @@ class Comment(entities.Entity['BackendComment']):
     def objects(cls: Type['Comment']) -> CommentCollection:  # type: ignore[misc] # pylint: disable=no-self-argument
         return CommentCollection.get_cached(cls, get_manager().get_profile_storage())
 
-    def __init__(self, node: 'Node', user: 'User', content: Optional[str] = None, backend: Optional['Backend'] = None):
+    def __init__(
+        self, node: 'Node', user: 'User', content: Optional[str] = None, backend: Optional['StorageBackend'] = None
+    ):
         """Create a Comment for a given node and user
 
         :param node: a Node instance

aiida/orm/computers.py

@@ -21,7 +21,7 @@
 
 if TYPE_CHECKING:
     from aiida.orm import AuthInfo, User
-    from aiida.orm.implementation import Backend, BackendComputer
+    from aiida.orm.implementation import BackendComputer, StorageBackend
     from aiida.schedulers import Scheduler
     from aiida.transports import Transport
 
@@ -89,7 +89,7 @@ def __init__(  # pylint: disable=too-many-arguments
         transport_type: str = '',
         scheduler_type: str = '',
         workdir: str = None,
-        backend: Optional['Backend'] = None,
+        backend: Optional['StorageBackend'] = None,
     ) -> None:
         """Construct a new computer."""
         backend = backend or get_manager().get_profile_storage()

aiida/orm/entities.py

@@ -21,7 +21,7 @@
 from aiida.manage import get_manager
 
 if TYPE_CHECKING:
-    from aiida.orm.implementation import Backend, BackendEntity
+    from aiida.orm.implementation import BackendEntity, StorageBackend
     from aiida.orm.querybuilder import FilterType, OrderByType, QueryBuilder
 
 __all__ = ('Entity', 'Collection', 'EntityAttributesMixin', 'EntityExtrasMixin', 'EntityTypes')
@@ -56,28 +56,28 @@ def _entity_base_cls() -> Type[EntityType]:
 
     @classmethod
     @lru_cache(maxsize=100)
-    def get_cached(cls, entity_class: Type[EntityType], backend: 'Backend'):
+    def get_cached(cls, entity_class: Type[EntityType], backend: 'StorageBackend'):
         """Get the cached collection instance for the given entity class and backend.
 
         :param backend: the backend instance to get the collection for
         """
-        from aiida.orm.implementation import Backend
-        type_check(backend, Backend)
+        from aiida.orm.implementation import StorageBackend
+        type_check(backend, StorageBackend)
         return cls(entity_class, backend=backend)
 
-    def __init__(self, entity_class: Type[EntityType], backend: Optional['Backend'] = None) -> None:
+    def __init__(self, entity_class: Type[EntityType], backend: Optional['StorageBackend'] = None) -> None:
         """ Construct a new entity collection.
 
         :param entity_class: the entity type e.g. User, Computer, etc
         :param backend: the backend instance to get the collection for, or use the default
         """
-        from aiida.orm.implementation import Backend
-        type_check(backend, Backend, allow_none=True)
+        from aiida.orm.implementation import StorageBackend
+        type_check(backend, StorageBackend, allow_none=True)
         assert issubclass(entity_class, self._entity_base_cls())
         self._backend = backend or get_manager().get_profile_storage()
         self._entity_type = entity_class
 
-    def __call__(self: CollectionType, backend: 'Backend') -> CollectionType:
+    def __call__(self: CollectionType, backend: 'StorageBackend') -> CollectionType:
         """Get or create a cached collection using a new backend."""
         if backend is self._backend:
             return self
@@ -89,7 +89,7 @@ def entity_type(self) -> Type[EntityType]:
         return self._entity_type
 
     @property
-    def backend(self) -> 'Backend':
+    def backend(self) -> 'StorageBackend':
         """Return the backend."""
         return self._backend
 
@@ -244,7 +244,7 @@ def is_stored(self) -> bool:
         return self._backend_entity.is_stored
 
     @property
-    def backend(self) -> 'Backend':
+    def backend(self) -> 'StorageBackend':
         """Get the backend for this entity"""
         return self._backend_entity.backend
 

aiida/orm/groups.py

@@ -20,7 +20,7 @@
 
 if TYPE_CHECKING:
     from aiida.orm import Node, User
-    from aiida.orm.implementation import Backend, BackendGroup
+    from aiida.orm.implementation import BackendGroup, StorageBackend
 
 __all__ = ('Group', 'AutoGroup', 'ImportGroup', 'UpfFamily')
 
@@ -127,7 +127,7 @@ def __init__(
         user: Optional['User'] = None,
         description: str = '',
         type_string: Optional[str] = None,
-        backend: Optional['Backend'] = None
+        backend: Optional['StorageBackend'] = None
     ):
         """
         Create a new group. Either pass a dbgroup parameter, to reload

aiida/orm/implementation/__init__.py

@@ -7,27 +7,26 @@
 # For further information on the license, see the LICENSE.txt file        #
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
-"""Module with the implementations of the various backend entities for various database backends."""
+"""Module containing the backend entity abstracts for storage backends."""
 
 # AUTO-GENERATED
 
 # yapf: disable
 # pylint: disable=wildcard-import
 
 from .authinfos import *
-from .backends import *
 from .comments import *
 from .computers import *
 from .entities import *
 from .groups import *
 from .logs import *
 from .nodes import *
 from .querybuilder import *
+from .storage_backend import *
 from .users import *
 from .utils import *
 
 __all__ = (
-    'Backend',
     'BackendAuthInfo',
     'BackendAuthInfoCollection',
     'BackendCollection',
@@ -47,6 +46,7 @@
     'BackendUser',
     'BackendUserCollection',
     'EntityType',
+    'StorageBackend',
     'clean_value',
     'validate_attribute_extra_key',
 )

aiida/orm/implementation/entities.py

@@ -12,7 +12,7 @@
 from typing import TYPE_CHECKING, Any, ClassVar, Dict, Generic, Iterable, List, Tuple, Type, TypeVar
 
 if TYPE_CHECKING:
-    from aiida.orm.implementation import Backend
+    from aiida.orm.implementation import StorageBackend
 
 __all__ = ('BackendEntity', 'BackendCollection', 'EntityType', 'BackendEntityExtrasMixin')
 
@@ -22,11 +22,11 @@
 class BackendEntity(abc.ABC):
     """An first-class entity in the backend"""
 
-    def __init__(self, backend: 'Backend', **kwargs: Any):  # pylint: disable=unused-argument
+    def __init__(self, backend: 'StorageBackend', **kwargs: Any):  # pylint: disable=unused-argument
         self._backend = backend
 
     @property
-    def backend(self) -> 'Backend':
+    def backend(self) -> 'StorageBackend':
         """Return the backend this entity belongs to
 
         :return: the backend instance
@@ -74,15 +74,15 @@ class BackendCollection(Generic[EntityType]):
 
     ENTITY_CLASS: ClassVar[Type[EntityType]]  # type: ignore[misc]
 
-    def __init__(self, backend: 'Backend'):
+    def __init__(self, backend: 'StorageBackend'):
         """
         :param backend: the backend this collection belongs to
         """
         assert issubclass(self.ENTITY_CLASS, BackendEntity), 'Must set the ENTRY_CLASS class variable to an entity type'
         self._backend = backend
 
     @property
-    def backend(self) -> 'Backend':
+    def backend(self) -> 'StorageBackend':
         """Return the backend."""
         return self._backend