Fix loading of database environment in ReadTheDocs environment (#3374)

The environment of ReadTheDocs does not actually have a database and so
the typical configuration and profile loading had to be hotwired. A
special global flag `IN_RTD_DOC_MODE` in `aiida.manage.configuration`
would circumvent loading an actual configuration for a dummy one. Since
the actual database environment never had to be loaded, the fact that
this contained non-sensical database connection parameters was
irrelevant. However, recently the documentation was changed to use the
Sphinx extension to document the `CalcJob` class. This now requires
loading the `aiida.orm` module, which in turn requires the database
environment being loaded.

The fix is to actually load the database environment through the backend
manager, but without performing the schema check. This will work despite
the unusable database configuration because no actual connection is
established in this way.

aiida/manage/configuration/__init__.py

@@ -24,7 +24,10 @@
 # This is used (and should be set to true) for the correct compilation of the documentation on readthedocs
 IN_RT_DOC_MODE = False
 
-__all__ = (config.__all__ + options.__all__ + profile.__all__ + ('get_config', 'get_config_option', 'load_profile'))
+__all__ = (
+    config.__all__ + options.__all__ + profile.__all__ +
+    ('get_config', 'get_config_option', 'load_profile', 'reset_config')
+)
 
 
 def load_profile(profile=None):
@@ -89,23 +92,25 @@ def load_config(create=False):
         # The following is a dummy config.json configuration that it is used for the
         # proper compilation of the documentation on readthedocs.
         from aiida.manage.external.postgres import DEFAULT_DBINFO
-        return ({
-            'default_profile': 'default',
-            'profiles': {
-                'default': {
-                    'AIIDADB_ENGINE': 'postgresql_psycopg2',
-                    'AIIDADB_BACKEND': 'django',
-                    'AIIDADB_HOST': DEFAULT_DBINFO['host'],
-                    'AIIDADB_PORT': DEFAULT_DBINFO['port'],
-                    'AIIDADB_NAME': 'aiidadb',
-                    'AIIDADB_PASS': '123',
-                    'default_user_email': 'aiida@epfl.ch',
-                    'TIMEZONE': 'Europe/Zurich',
-                    'AIIDADB_REPOSITORY_URI': 'file:///repository',
-                    'AIIDADB_USER': 'aiida'
+        return Config(
+            '/dev/null', {
+                'default_profile': 'default',
+                'profiles': {
+                    'default': {
+                        'AIIDADB_ENGINE': 'postgresql_psycopg2',
+                        'AIIDADB_BACKEND': 'django',
+                        'AIIDADB_HOST': DEFAULT_DBINFO['host'],
+                        'AIIDADB_PORT': DEFAULT_DBINFO['port'],
+                        'AIIDADB_NAME': 'aiidadb',
+                        'AIIDADB_PASS': '123',
+                        'default_user_email': 'aiida@epfl.ch',
+                        'TIMEZONE': 'Europe/Zurich',
+                        'AIIDADB_REPOSITORY_URI': 'file:///tmp/repository',
+                        'AIIDADB_USER': 'aiida'
+                    }
                 }
             }
-        })
+        )
 
     filepath = os.path.join(AIIDA_CONFIG_FOLDER, DEFAULT_CONFIG_FILE_NAME)
 
@@ -149,6 +154,16 @@ def reset_profile():
     BACKEND_UUID = None
 
 
+def reset_config():
+    """Reset the globally loaded config.
+
+    .. warning:: This is experimental functionality and should for now be used only internally. If the reset is unclean
+        weird unknown side-effects may occur that end up corrupting or destroying data.
+    """
+    global CONFIG
+    CONFIG = None
+
+
 def get_config(create=False):
     """Return the current configuration.
 
@@ -160,7 +175,6 @@ def get_config(create=False):
     :param create: if True, will create the configuration file if it does not already exist
     :type create: bool
 
-
     :return: the config
     :rtype: :class:`~aiida.manage.configuration.config.Config`
     :raises aiida.common.ConfigurationError: if the configuration file could not be found, read or deserialized

aiida/manage/configuration/config.py

@@ -200,7 +200,9 @@ def get_profile(self, name=None):
         from aiida.common import exceptions
 
         if not name and not self.default_profile_name:
-            raise exceptions.ProfileConfigurationError('no default profile defined')
+            raise exceptions.ProfileConfigurationError(
+                'no default profile defined: {}\n{}'.format(self._default_profile, self.dictionary)
+            )
 
         if not name:
             name = self.default_profile_name

docs/requirements_for_rtd.txt

@@ -46,6 +46,7 @@ pgtest==1.3.1
 pika==1.0.0
 plumpy==0.14.2
 psutil==5.5.1
+psycopg2-binary==2.8
 pyblake2==1.1.2; python_version<'3.6'
 pymatgen<=2018.12.12
 pyparsing==2.3.1

docs/source/conf.py

@@ -257,10 +257,21 @@
     load_profile(config.default_profile_name)
     get_manager().get_backend()
 else:
-    # Back-end settings for readthedocs online documentation.
     from aiida.manage import configuration
+    from aiida.manage.configuration import load_profile, reset_config
+    from aiida.manage.manager import get_manager
+
+    # Set the global variable to trigger shortcut behavior in `aiida.manager.configuration.load_config`
     configuration.IN_RT_DOC_MODE = True
-    configuration.BACKEND = 'django'
+
+    # First need to reset the config, because an empty one will have been loaded when `aiida` module got imported
+    reset_config()
+
+    # Load the profile: this will first load the config, which will be the dummy one for RTD purposes
+    load_profile()
+
+    # Finally load the database backend but without checking the schema because there is no actual database
+    get_manager()._load_backend(schema_check=False)
 
 
 def run_apidoc(_):

docs/update_req_for_rtd.py

@@ -33,8 +33,7 @@ def update_req_for_rtd(pre_commit):
 
     extras = setup_json['extras_require']
     reqs = set(extras['testing'] + extras['docs'] + extras['rest'] + extras['atomic_tools'] +
-               # To avoid that it requires also the postgres libraries
-               [p for p in setup_json['install_requires'] if not p.startswith('psycopg2')])
+        setup_json['install_requires'])
     reqs_str = '\n'.join(sorted(reqs))
 
     basename = 'requirements_for_rtd.txt'