Implement support for process classes to QueryBuilder.append (#2421)

Any `Process` class will have an associated `ProcessNode` sub class
associated with it that it will use to store its provenance when ran. In
addition with the `process_type`, determined by its specific process sub
class entry point, defines a set of nodes that correspond to that
process class.

For example: the `ArithmeticAddCalculation` will use a `CalcJobNode` to
store its executions and has a process type according to its entry point:

    `aiida.calculations:arithmetic.add`

This commit implements the functionality necessary that allows a user to
append the `ArithmeticAddCalculation` process class to a query builder
instance directly.

.pre-commit-config.yaml

@@ -110,7 +110,6 @@
         aiida/backends/tests/orm/data/test_remote.py|
         aiida/backends/tests/orm/utils/test_loaders.py|
         aiida/backends/tests/test_parsers.py|
-        aiida/backends/tests/test_query.py|
         aiida/backends/tests/test_tcodexporter.py|
         aiida/backends/tests/test_caching_config.py|
         aiida/backends/tests/test_plugin_loader.py|

aiida/backends/djsite/db/subtests/test_query.py

@@ -34,57 +34,49 @@ def test_clsf_django(self):
         with self.assertRaises(DbContentError):
             qb._get_ormclass(None, '.')
 
-        for cls, clstype, query_type_string in (
+        for cls, classifiers in (
                 qb._get_ormclass(StructureData, None),
                 qb._get_ormclass(None, 'data.structure.StructureData.'),
         ):
-            self.assertEqual(clstype, 'data.structure.StructureData.')
+            self.assertEqual(classifiers['ormclass_type_string'], 'data.structure.StructureData.')
             self.assertTrue(issubclass(cls, DbNode))
-            self.assertEqual(clstype, 'data.structure.StructureData.')
-            self.assertEqual(query_type_string,
-                             StructureData._query_type_string)
 
-        for cls, clstype, query_type_string in (
+        for cls, classifiers in (
                 qb._get_ormclass(DbNode, None),
         ):
-            self.assertEqual(clstype, Node._plugin_type_string)
-            self.assertEqual(query_type_string, Node._query_type_string)
+            self.assertEqual(classifiers['ormclass_type_string'], Node._plugin_type_string)
             self.assertTrue(issubclass(cls, DbNode))
 
-        for cls, clstype, query_type_string in (
+        for cls, classifiers in (
                 qb._get_ormclass(DbGroup, None),
                 qb._get_ormclass(Group, None),
                 qb._get_ormclass(None, 'group'),
                 qb._get_ormclass(None, 'Group'),
         ):
-            self.assertEqual(clstype, 'group')
-            self.assertEqual(query_type_string, None)
+            self.assertEqual(classifiers['ormclass_type_string'], 'group')
             self.assertTrue(issubclass(cls, DbGroup))
 
-        for cls, clstype, query_type_string in (
+        for cls, classifiers in (
                 qb._get_ormclass(DbUser, None),
                 qb._get_ormclass(DbUser, None),
                 qb._get_ormclass(None, "user"),
                 qb._get_ormclass(None, "User"),
         ):
-            self.assertEqual(clstype, 'user')
-            self.assertEqual(query_type_string, None)
+            self.assertEqual(classifiers['ormclass_type_string'], 'user')
             self.assertTrue(issubclass(cls, DbUser))
 
-        for cls, clstype, query_type_string in (
+        for cls, classifiers in (
                 qb._get_ormclass(DbComputer, None),
                 qb._get_ormclass(Computer, None),
                 qb._get_ormclass(None, 'computer'),
                 qb._get_ormclass(None, 'Computer'),
         ):
-            self.assertEqual(clstype, 'computer')
-            self.assertEqual(query_type_string, None)
+            self.assertEqual(classifiers['ormclass_type_string'], 'computer')
             self.assertTrue(issubclass(cls, DbComputer))
 
-        for cls, clstype, query_type_string in (
+        for cls, classifiers in (
                 qb._get_ormclass(Data, None),
                 qb._get_ormclass(None, 'data.Data.'),
         ):
-            self.assertEqual(clstype, Data._plugin_type_string)
-            self.assertEqual(query_type_string, Data._query_type_string)
+            self.assertEqual(classifiers['ormclass_type_string'], Data._plugin_type_string)
             self.assertTrue(issubclass(cls, DbNode))

aiida/backends/djsite/db/testbase.py

@@ -13,8 +13,6 @@
 from __future__ import division
 from __future__ import print_function
 from __future__ import absolute_import
-import shutil
-import os
 
 from aiida.backends.testimplbase import AiidaTestImplementation
 from aiida.orm.implementation.django.backend import DjangoBackend
@@ -56,20 +54,7 @@ def clean_db(self):
         models.DbComputer.objects.all().delete()
         models.DbGroup.objects.all().delete()
 
-    # Note this is has to be a normal method, not a class method
     def tearDownClass_method(self):
-        from aiida.settings import REPOSITORY_PATH
-        from aiida.common.setup import TEST_KEYWORD
-        from aiida.common.exceptions import InvalidOperation
-
-        base_repo_path = os.path.basename(os.path.normpath(REPOSITORY_PATH))
-        if TEST_KEYWORD not in base_repo_path:
-            raise InvalidOperation("Be careful. The repository for the tests "
-                                   "is not a test repository. I will not "
-                                   "empty the database and I will not delete "
-                                   "the repository. Repository path: "
-                                   "{}".format(REPOSITORY_PATH))
-
-        # I clean the test repository
-        shutil.rmtree(REPOSITORY_PATH, ignore_errors=True)
-        os.makedirs(REPOSITORY_PATH)
+        """
+        Backend-specific tasks for tearing down the test environment.
+        """

aiida/backends/sqlalchemy/tests/test_query.py

@@ -28,14 +28,12 @@ def test_clsf_sqla(self):
         from aiida.orm.querybuilder import QueryBuilder
 
         qb = QueryBuilder()
-        for AiidaCls, ORMCls, typestr in zip(
+        for AiidaCls, ORMCls in zip(
                 (Group, User, Computer, Node, Data, ProcessNode),
-                (DbGroup, DbUser, DbComputer, DbNode, DbNode, DbNode),
-                (None, None, None, Node._query_type_string, Data._query_type_string, ProcessNode._query_type_string)):
-            cls, clstype, query_type_string = qb._get_ormclass(AiidaCls, None)
+                (DbGroup, DbUser, DbComputer, DbNode, DbNode, DbNode)):
+            cls, classifiers = qb._get_ormclass(AiidaCls, None)
 
             self.assertEqual(cls, ORMCls)
-            self.assertEqual(query_type_string, typestr)
 
 
 class QueryBuilderLimitOffsetsTestSQLA(AiidaTestCase):

aiida/backends/sqlalchemy/tests/testbase.py

@@ -112,19 +112,8 @@ def clean_db(self):
         self.test_session.commit()
 
     def tearDownClass_method(self):
-        from aiida.settings import REPOSITORY_PATH
-        from aiida.common.setup import TEST_KEYWORD
-        from aiida.common.exceptions import InvalidOperation
-        if TEST_KEYWORD not in REPOSITORY_PATH:
-            raise InvalidOperation("Be careful. The repository for the tests "
-                                   "is not a test repository. I will not "
-                                   "empty the database and I will not delete "
-                                   "the repository. Repository path: "
-                                   "{}".format(REPOSITORY_PATH))
-
+        """
+        Backend-specific tasks for tearing down the test environment.
+        """
         self.test_session.close()
         self.test_session = None
-
-        # I clean the test repository
-        shutil.rmtree(REPOSITORY_PATH, ignore_errors=True)
-        os.makedirs(REPOSITORY_PATH)

aiida/backends/testbase.py

@@ -127,20 +127,46 @@ def insert_data(cls):
 
     @classmethod
     def clean_db(cls):
+        """Clean up database and reset caches.
+
+        Resets AiiDA manager cache, which could otherwise be left in an inconsistent state when cleaning the database.
+        """
+        from aiida.common.exceptions import InvalidOperation
+
         # Note: this will raise an exception, that will be seen as a test
         # failure. To be safe, you should do the same check also in the tearDownClass
         # to avoid that it is run
         check_if_tests_can_run()
 
-        from aiida.common.exceptions import InvalidOperation
-
         if not cls._class_was_setup:
             raise InvalidOperation("You cannot call clean_db before running the setUpClass")
 
         cls.__backend_instance.clean_db()
-        # Make sure to reset the backend when cleaning the database
+
+        # Reset AiiDA manager cache
         reset_manager()
 
+    @classmethod
+    def clean_repository(cls):
+        """
+        Cleans up file repository.
+        """
+        from aiida.settings import REPOSITORY_PATH
+        from aiida.common.setup import TEST_KEYWORD
+        from aiida.common.exceptions import InvalidOperation
+        import shutil
+
+        base_repo_path = os.path.basename(os.path.normpath(REPOSITORY_PATH))
+        if TEST_KEYWORD not in base_repo_path:
+            raise InvalidOperation("Warning: The repository folder {} does not "
+                                   "seem to belong to a test profile and will therefore not be deleted.\n"
+                                   "Full repository path: "
+                                   "{}".format(base_repo_path, REPOSITORY_PATH))
+
+        # Clean the test repository
+        shutil.rmtree(REPOSITORY_PATH, ignore_errors=True)
+        os.makedirs(REPOSITORY_PATH)
+
     @classproperty
     def computer(cls):
         """
@@ -161,6 +187,7 @@ def tearDownClass(cls, *args, **kwargs):
         # if this is not a test profile
         check_if_tests_can_run()
         cls.clean_db()
+        cls.clean_repository()
         cls.__backend_instance.tearDownClass_method(*args, **kwargs)
 
     def assertClickSuccess(self, cli_result):

aiida/backends/testimplbase.py

@@ -51,26 +51,24 @@ def setUpClass_method(self):
         You have also to set a self.computer and a self.user_email as explained in the docstring of the
         AiidaTestImplemention docstring.
         """
-        pass
 
     def setUp_method(self):
         pass
 
     def tearDown_method(self):
         pass
 
+    @abstractmethod
     def tearDownClass_method(self):
         """
-        This class implements the tear down methods (e.g. cleans up the DB).
+        Backend-specific tasks for tearing down the test environment.
         """
-        pass
 
     @abstractmethod
     def clean_db(self):
         """
         This method implements the logic to fully clean the DB.
         """
-        pass
 
     def insert_data(self):
         """