Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Faster aiida imports #6092

Closed
wants to merge 52 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
52 commits
Select commit Hold shift + click to select a range
ce8f249
Lazily define `_plugin_type_string` and `_query_type_string of `Node`
sphuber Jul 7, 2023
1433071
Lazy import object_diskstore
danielhollas Jul 8, 2023
872856f
WIP: Lazy import numpy
danielhollas Jul 8, 2023
9c81e37
Delay import of jsonschema and requests
danielhollas Jul 9, 2023
286f752
Avoid importing aiida.transports
danielhollas Jul 9, 2023
dbfbd90
Fix numpy import
danielhollas Jul 9, 2023
a14b110
Revert incomplete orm/groups.py change
danielhollas Jul 9, 2023
d86c7b9
Lazily import yaml, -20ms
danielhollas Jul 9, 2023
a51cdff
Delay plumpy import, -90ms for import.aiida!
danielhollas Jul 11, 2023
edb82ce
Lazily define __type_string in orm.Group, another 30ms
danielhollas Jul 11, 2023
364070f
Lazy import importlib_metadata, -20ms for aiida.cmdline and aiida, no…
danielhollas Jul 11, 2023
f0dc6b0
Do not import aiida.cmdline from aiida.orm, saves 70ms!
danielhollas Jul 11, 2023
8ccd938
Lazy import paramiko, -65ms for aiida.transports
danielhollas Jul 11, 2023
9f8f1af
Use cache instead of lru_cache(maxsize=1)
danielhollas Jul 11, 2023
1b24c39
Lazy urllib.*, -10ms to everything
danielhollas Jul 11, 2023
0b27a2a
Lazy tabulate, -5ms to everything
danielhollas Jul 11, 2023
824b858
Speed up import aiida.cmdline by not import aiida.cmdline.params.options
danielhollas Jul 11, 2023
f0db0f5
Do not import pgsu, -30ms to everything except verdi
danielhollas Jul 11, 2023
2b06b3d
Switch to fastjsonschema for much faster import speed
danielhollas Jul 12, 2023
fc974d3
Cache importlib_metadata.entry_points().select()
danielhollas Jul 12, 2023
2f543de
Fix fastjsonschema
danielhollas Jul 12, 2023
41f8e86
Temporarily format fastjsonschema file
danielhollas Jul 12, 2023
56c7cbe
Ignore mypy errors in fastjsonschema code
danielhollas Jul 12, 2023
a3e0ebd
try appeasing mypy
danielhollas Jul 12, 2023
f2de5cf
simplify rmq_launcher
danielhollas Jul 12, 2023
a93937a
Try improving types in entry_point.py
danielhollas Jul 12, 2023
4fb0438
Try appeasing pylint
danielhollas Jul 12, 2023
bb018b8
Skip not needed
danielhollas Jul 12, 2023
2d09c5b
Fix type in entry_point.py
danielhollas Jul 12, 2023
0199c56
Fix import in conftest
danielhollas Jul 12, 2023
3f55405
mypy, sigh
danielhollas Jul 12, 2023
aac7540
Fix test now?
danielhollas Jul 12, 2023
9240c7a
Bring back numpy
danielhollas Jul 12, 2023
774e6aa
Fix eps_select
danielhollas Jul 13, 2023
fafaa7c
Temporarily import EntryPoint
danielhollas Jul 14, 2023
9f2efbf
Comment out EntryPoints cache
danielhollas Jul 18, 2023
7498473
Try fixing test_creation_unregistered test
danielhollas Jul 19, 2023
7822680
Revert EntryPoint import in aiida.plugins
danielhollas Jul 19, 2023
93faf93
Fix entry_point.py
danielhollas Jul 19, 2023
c1851bc
Vendor the fastjsonschema exceptions to avoid extra import
danielhollas Jul 26, 2023
ae75207
Fix
danielhollas Jul 26, 2023
138d95b
Merge branch 'main' into faster-import
danielhollas Aug 31, 2023
be8119c
Revert "Lazy tabulate, -5ms to everything"
danielhollas Sep 4, 2023
2ac4faf
Fix import
danielhollas Sep 4, 2023
08140cd
Revert "WIP: Lazy import numpy"
danielhollas Sep 4, 2023
336db39
Merge branch 'main' into faster-import
danielhollas Sep 4, 2023
1208fae
Merge branch 'main' into faster-import
danielhollas Sep 5, 2023
a55f04b
Merge branch 'main' into faster-import
danielhollas Sep 12, 2023
3c65d78
do not use type strings
danielhollas Sep 12, 2023
9bc86b2
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Sep 12, 2023
1a311c4
Simplify factories.py
danielhollas Sep 12, 2023
0ba095e
fix conda build
danielhollas Sep 12, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions aiida/cmdline/groups/dynamic.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@
from aiida.common import exceptions
from aiida.plugins.entry_point import ENTRY_POINT_GROUP_FACTORY_MAPPING, get_entry_point_names

from ..params import options
from ..params.options.interactive import InteractiveOption
from .verdi import VerdiCommandGroup

__all__ = ('DynamicEntryPointCommandGroup',)
Expand Down Expand Up @@ -93,6 +91,7 @@ def create_options(self, entry_point):

:param entry_point: The entry point.
"""
from ..params import options

def apply_options(func):
"""Decorate the command function with the appropriate options for the given entry point."""
Expand Down Expand Up @@ -125,6 +124,8 @@ def list_options(self, entry_point):
@staticmethod
def create_option(name, spec):
"""Create a click option from a name and a specification."""
from ..params.options.interactive import InteractiveOption

spec = copy.deepcopy(spec)

is_flag = spec.pop('is_flag', False)
Expand Down
4 changes: 2 additions & 2 deletions aiida/cmdline/groups/verdi.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@
from aiida.common.extendeddicts import AttributeDict
from aiida.manage.configuration import get_config

from ..params import options

__all__ = ('VerdiCommandGroup',)

GIU = (
Expand Down Expand Up @@ -61,6 +59,8 @@ class VerdiCommandGroup(click.Group):
@staticmethod
def add_verbosity_option(cmd: click.Command):
"""Apply the ``verbosity`` option to the command, which is common to all ``verdi`` commands."""
from ..params import options

# Only apply the option if it hasn't been already added in a previous call.
if 'verbosity' not in [param.name for param in cmd.params]:
cmd = options.VERBOSITY()(cmd)
Expand Down
3 changes: 2 additions & 1 deletion aiida/cmdline/params/types/plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
import functools

import click
from importlib_metadata import EntryPoint

from aiida.common import exceptions
from aiida.plugins import factories
Expand Down Expand Up @@ -236,6 +235,8 @@ def convert(self, value, param, ctx):
Convert the string value to an entry point instance, if the value can be successfully parsed
into an actual entry point. Will raise click.BadParameter if validation fails.
"""
from importlib_metadata import EntryPoint

# If the value is already of the expected return type, simply return it. This behavior is new in `click==8.0`:
# https://click.palletsprojects.com/en/8.0.x/parameters/#implementing-custom-types
if isinstance(value, EntryPoint):
Expand Down
15 changes: 7 additions & 8 deletions aiida/manage/configuration/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
###########################################################################
"""Module that defines the configuration file of an AiiDA instance and functions to create and load it."""
import codecs
from functools import lru_cache
import functools
from importlib.resources import files
import json
import os
Expand All @@ -28,7 +28,7 @@
SCHEMA_FILE = 'config-v9.schema.json'


@lru_cache(1)
@functools.cache
def config_schema() -> Dict[str, Any]:
"""Return the configuration schema."""
return json.loads(files(schema_module).joinpath(SCHEMA_FILE).read_text(encoding='utf8'))
Expand Down Expand Up @@ -124,13 +124,12 @@ def _backup(cls, filepath):
@staticmethod
def validate(config: dict, filepath: Optional[str] = None):
"""Validate a configuration dictionary."""
import jsonschema
from .schema.fastjsonschema_validate_v9 import JsonSchemaValueException, validate

try:
jsonschema.validate(instance=config, schema=config_schema())
except jsonschema.ValidationError as error:
raise ConfigValidationError(
message=error.message, keypath=error.path, schema=error.schema, filepath=filepath
)
validate(data=config)
except JsonSchemaValueException as error:
raise ConfigValidationError(message=error.message, keypath=error.path, schema=None, filepath=filepath)

def __init__(self, filepath: str, config: dict, validate: bool = True):
"""Instantiate a configuration object from a configuration dictionary and its filepath.
Expand Down
8 changes: 4 additions & 4 deletions aiida/manage/configuration/options.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ def validate(self, value: Any, cast: bool = True) -> Any:

"""
# pylint: disable=too-many-branches
import jsonschema
import fastjsonschema

from aiida.manage.caching import _validate_identifier_pattern

Expand Down Expand Up @@ -90,9 +90,9 @@ def validate(self, value: Any, cast: bool = True) -> Any:
pass

try:
jsonschema.validate(instance=value, schema=self.schema)
except jsonschema.ValidationError as exc:
raise ConfigValidationError(message=exc.message, keypath=[self.name, *(exc.path or [])], schema=exc.schema)
fastjsonschema.validate(data=value, definition=self.schema)
except fastjsonschema.JsonSchemaException as exc:
raise ConfigValidationError(message=exc.message, keypath=[self.name, *(exc.path or [])], schema=self.schema) # pylint: disable=no-member

# special caching validation
if self.name in ('caching.enabled_for', 'caching.disabled_for'):
Expand Down
Loading