From d51aca03f3938148f96d7d33cdc85209c7412e04 Mon Sep 17 00:00:00 2001 From: Sebastiaan Huber Date: Fri, 21 Oct 2022 16:08:55 +0200 Subject: [PATCH 1/2] Mark relevant `Process` exit codes as `invalidates_cache=True` The `Process` and `CalcJob` classes define various exit codes, all of which will correspond to processes that should not be used as a source for caching. Therefore, these exit codes should properly declare `invalidate_cache=True` which will automatically prevent any node with that exit code from being cached from. The `ArithmeticAddCalculation` and `TemplatereplacerCalculation plugins are also updated to properly mark exit codes that should not be cached from. --- aiida/calculations/arithmetic/add.py | 8 ++- aiida/calculations/templatereplacer.py | 68 +++++++++++++++------- aiida/engine/processes/calcjobs/calcjob.py | 20 ++++--- aiida/engine/processes/process.py | 19 ++++-- 4 files changed, 80 insertions(+), 35 deletions(-) diff --git a/aiida/calculations/arithmetic/add.py b/aiida/calculations/arithmetic/add.py index fbf0affa6f..b3d327b19c 100644 --- a/aiida/calculations/arithmetic/add.py +++ b/aiida/calculations/arithmetic/add.py @@ -34,8 +34,12 @@ def define(cls, spec: CalcJobProcessSpec): spec.inputs['metadata']['options']['output_filename'].default = 'aiida.out' spec.inputs['metadata']['options']['resources'].default = {'num_machines': 1, 'num_mpiprocs_per_machine': 1} # start exit codes - marker for docs - spec.exit_code(310, 'ERROR_READING_OUTPUT_FILE', message='The output file could not be read.') - spec.exit_code(320, 'ERROR_INVALID_OUTPUT', message='The output file contains invalid output.') + spec.exit_code( + 310, 'ERROR_READING_OUTPUT_FILE', invalidates_cache=True, message='The output file could not be read.' + ) + spec.exit_code( + 320, 'ERROR_INVALID_OUTPUT', invalidates_cache=True, message='The output file contains invalid output.' + ) spec.exit_code(410, 'ERROR_NEGATIVE_NUMBER', message='The sum of the operands is a negative number.') # end exit codes - marker for docs diff --git a/aiida/calculations/templatereplacer.py b/aiida/calculations/templatereplacer.py index 5933c9ebab..4e6e8de1e8 100644 --- a/aiida/calculations/templatereplacer.py +++ b/aiida/calculations/templatereplacer.py @@ -61,29 +61,47 @@ class TemplatereplacerCalculation(CalcJob): @classmethod def define(cls, spec): - # yapf: disable super().define(spec) spec.inputs['metadata']['options']['parser_name'].default = 'core.templatereplacer' - spec.input('template', valid_type=orm.Dict, - help='A template for the input file.') - spec.input('parameters', valid_type=orm.Dict, required=False, - help='Parameters used to replace placeholders in the template.') + spec.input('template', valid_type=orm.Dict, help='A template for the input file.') + spec.input( + 'parameters', + valid_type=orm.Dict, + required=False, + help='Parameters used to replace placeholders in the template.' + ) spec.input_namespace('files', valid_type=(orm.RemoteData, orm.SinglefileData), required=False, dynamic=True) spec.output('output_parameters', valid_type=orm.Dict, required=True) spec.default_output_node = 'output_parameters' - spec.exit_code(101, 'ERROR_NO_TEMPORARY_RETRIEVED_FOLDER', - message='The temporary retrieved folder data node could not be accessed.') - spec.exit_code(105, 'ERROR_NO_OUTPUT_FILE_NAME_DEFINED', - message='The `template` input node did not specify the key `output_file_name`.') - spec.exit_code(110, 'ERROR_READING_OUTPUT_FILE', - message='The output file could not be read from the retrieved folder.') - spec.exit_code(111, 'ERROR_READING_TEMPORARY_RETRIEVED_FILE', - message='A temporary retrieved file could not be read from the temporary retrieved folder.') - spec.exit_code(120, 'ERROR_INVALID_OUTPUT', - message='The output file contains invalid output.') - + spec.exit_code( + 101, + 'ERROR_NO_TEMPORARY_RETRIEVED_FOLDER', + invalidates_cache=True, + message='The temporary retrieved folder data node could not be accessed.' + ) + spec.exit_code( + 105, + 'ERROR_NO_OUTPUT_FILE_NAME_DEFINED', + invalidates_cache=True, + message='The `template` input node did not specify the key `output_file_name`.' + ) + spec.exit_code( + 110, + 'ERROR_READING_OUTPUT_FILE', + invalidates_cache=True, + message='The output file could not be read from the retrieved folder.' + ) + spec.exit_code( + 111, + 'ERROR_READING_TEMPORARY_RETRIEVED_FILE', + invalidates_cache=True, + message='A temporary retrieved file could not be read from the temporary retrieved folder.' + ) + spec.exit_code( + 120, 'ERROR_INVALID_OUTPUT', invalidates_cache=True, message='The output file contains invalid output.' + ) def prepare_for_submission(self, folder): """ @@ -113,7 +131,8 @@ def prepare_for_submission(self, folder): if template: raise exceptions.InputValidationError( - f'The following keys could not be used in the template node: {template.keys()}') + f'The following keys could not be used in the template node: {template.keys()}' + ) try: validate_list_of_string_tuples(files_to_copy, tuple_length=2) @@ -127,8 +146,9 @@ def prepare_for_submission(self, folder): try: fileobj = self.inputs.files[link_name] except AttributeError: - raise exceptions.InputValidationError('You are asking to copy a file link {}, ' - 'but there is no input link with such a name'.format(link_name)) + raise exceptions.InputValidationError( + f'You are asking to copy a file link {link_name}, but there is no input link with such a name' + ) if isinstance(fileobj, orm.SinglefileData): local_copy_list.append((fileobj.uuid, fileobj.filename, dest_rel_path)) elif isinstance(fileobj, orm.RemoteData): # can be a folder @@ -137,15 +157,19 @@ def prepare_for_submission(self, folder): raise exceptions.InputValidationError( 'If you ask to copy a file link {}, ' 'it must be either a SinglefileData or a RemoteData; it is instead of type {}'.format( - link_name, fileobj.__class__.__name__)) + link_name, fileobj.__class__.__name__ + ) + ) if input_file_name is not None and not input_file_template: raise exceptions.InputValidationError( - 'If you give an input_file_name, you must also specify a input_file_template') + 'If you give an input_file_name, you must also specify a input_file_template' + ) if input_through_stdin and input_file_name is None: raise exceptions.InputValidationError( - 'If you ask for input_through_stdin you have to specify a input_file_name') + 'If you ask for input_through_stdin you have to specify a input_file_name' + ) input_content = input_file_template.format(**parameters) if input_file_name: diff --git a/aiida/engine/processes/calcjobs/calcjob.py b/aiida/engine/processes/calcjobs/calcjob.py index 9460922459..30c7493e07 100644 --- a/aiida/engine/processes/calcjobs/calcjob.py +++ b/aiida/engine/processes/calcjobs/calcjob.py @@ -283,14 +283,20 @@ def define(cls, spec: CalcJobProcessSpec) -> None: # type: ignore[override] help='Files that are retrieved by the daemon will be stored in this node. By default the stdout and stderr ' 'of the scheduler will be added, but one can add more by specifying them in `CalcInfo.retrieve_list`.') - # Errors caused or returned by the scheduler - spec.exit_code(100, 'ERROR_NO_RETRIEVED_FOLDER', - message='The process did not have the required `retrieved` output.') - spec.exit_code(110, 'ERROR_SCHEDULER_OUT_OF_MEMORY', - message='The job ran out of memory.') - spec.exit_code(120, 'ERROR_SCHEDULER_OUT_OF_WALLTIME', - message='The job ran out of walltime.') # yapf: enable + # Errors caused or returned by the scheduler + spec.exit_code( + 100, + 'ERROR_NO_RETRIEVED_FOLDER', + invalidates_cache=True, + message='The process did not have the required `retrieved` output.' + ) + spec.exit_code( + 110, 'ERROR_SCHEDULER_OUT_OF_MEMORY', invalidates_cache=True, message='The job ran out of memory.' + ) + spec.exit_code( + 120, 'ERROR_SCHEDULER_OUT_OF_WALLTIME', invalidates_cache=True, message='The job ran out of walltime.' + ) @classproperty def spec_options(cls): # pylint: disable=no-self-argument diff --git a/aiida/engine/processes/process.py b/aiida/engine/processes/process.py index 1cc9e87fd6..770ed9fcce 100644 --- a/aiida/engine/processes/process.py +++ b/aiida/engine/processes/process.py @@ -115,10 +115,21 @@ def define(cls, spec: ProcessSpec) -> None: # type: ignore[override] ) spec.inputs.valid_type = orm.Data spec.inputs.dynamic = False # Settings a ``valid_type`` automatically makes it dynamic, so we reset it again - spec.exit_code(1, 'ERROR_UNSPECIFIED', message='The process has failed with an unspecified error.') - spec.exit_code(2, 'ERROR_LEGACY_FAILURE', message='The process failed with legacy failure mode.') - spec.exit_code(10, 'ERROR_INVALID_OUTPUT', message='The process returned an invalid output.') - spec.exit_code(11, 'ERROR_MISSING_OUTPUT', message='The process did not register a required output.') + spec.exit_code( + 1, 'ERROR_UNSPECIFIED', invalidates_cache=True, message='The process has failed with an unspecified error.' + ) + spec.exit_code( + 2, 'ERROR_LEGACY_FAILURE', invalidates_cache=True, message='The process failed with legacy failure mode.' + ) + spec.exit_code( + 10, 'ERROR_INVALID_OUTPUT', invalidates_cache=True, message='The process returned an invalid output.' + ) + spec.exit_code( + 11, + 'ERROR_MISSING_OUTPUT', + invalidates_cache=True, + message='The process did not register a required output.' + ) @classmethod def get_builder(cls) -> ProcessBuilder: From f11e9ea942b95dc6d402fa22e6bcf23e1f83040d Mon Sep 17 00:00:00 2001 From: Sebastiaan Huber Date: Fri, 21 Oct 2022 16:12:00 +0200 Subject: [PATCH 2/2] `TemplatereplacerCalculation`: Change exit codes to be in 300 range The exit codes where specified in the 100 range, which is reserved (by convention) for exit codes defined by the `CalcJob` class. In fact, some of them even overlapped with existing exit codes defined by `CalcJob`. There currently is no logic in `aiida-core` that will prevent or at least warn when existing exit codes get overwritten so this went unnoticed until now. Since the plugin is mostly used for internal testing purposes, there will not be an automated database migration to update existing exit codes. --- aiida/calculations/templatereplacer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/aiida/calculations/templatereplacer.py b/aiida/calculations/templatereplacer.py index 4e6e8de1e8..0e3da70496 100644 --- a/aiida/calculations/templatereplacer.py +++ b/aiida/calculations/templatereplacer.py @@ -76,31 +76,31 @@ def define(cls, spec): spec.default_output_node = 'output_parameters' spec.exit_code( - 101, + 301, 'ERROR_NO_TEMPORARY_RETRIEVED_FOLDER', invalidates_cache=True, message='The temporary retrieved folder data node could not be accessed.' ) spec.exit_code( - 105, + 305, 'ERROR_NO_OUTPUT_FILE_NAME_DEFINED', invalidates_cache=True, message='The `template` input node did not specify the key `output_file_name`.' ) spec.exit_code( - 110, + 310, 'ERROR_READING_OUTPUT_FILE', invalidates_cache=True, message='The output file could not be read from the retrieved folder.' ) spec.exit_code( - 111, + 311, 'ERROR_READING_TEMPORARY_RETRIEVED_FILE', invalidates_cache=True, message='A temporary retrieved file could not be read from the temporary retrieved folder.' ) spec.exit_code( - 120, 'ERROR_INVALID_OUTPUT', invalidates_cache=True, message='The output file contains invalid output.' + 320, 'ERROR_INVALID_OUTPUT', invalidates_cache=True, message='The output file contains invalid output.' ) def prepare_for_submission(self, folder):