From d620d3c744acfdf753af60f7e3735aa8d5cce6aa Mon Sep 17 00:00:00 2001 From: Sebastiaan Huber Date: Tue, 18 Oct 2022 12:19:55 +0200 Subject: [PATCH 1/3] Add the `ContainerizedCode` data plugin This implementation of the `AbstractCode` interface allows running a `CalcJob` within a container on a target computer. The data plugin stores the name of the container image, the executable within that container, the command with which the container should be launched and the `Computer` on which the container can be run. --- .github/config/add-singularity.yaml | 10 ++ .github/config/add.yaml | 1 - .github/config/doubler.yaml | 1 - .github/config/localhost.yaml | 1 + .../system_tests/test_containerized_code.py | 36 ++++ .github/workflows/nightly.yml | 3 + .github/workflows/setup.sh | 1 + .github/workflows/tests_nightly.sh | 1 + aiida/orm/__init__.py | 1 + aiida/orm/nodes/__init__.py | 1 + aiida/orm/nodes/data/__init__.py | 1 + aiida/orm/nodes/data/code/__init__.py | 2 + aiida/orm/nodes/data/code/containerized.py | 134 ++++++++++++++ docs/source/topics/data_types.rst | 163 ++++++++++++++---- pyproject.toml | 79 ++++----- tests/cmdline/commands/test_code.py | 21 ++- .../processes/calcjobs/test_calc_job.py | 38 +++- .../test_containerized_installed_code.sh | 6 + .../test_containerized_portable_code.sh | 6 + 19 files changed, 423 insertions(+), 83 deletions(-) create mode 100644 .github/config/add-singularity.yaml create mode 100644 .github/system_tests/test_containerized_code.py create mode 100644 aiida/orm/nodes/data/code/containerized.py create mode 100644 tests/engine/processes/calcjobs/test_calc_job/test_containerized_installed_code.sh create mode 100644 tests/engine/processes/calcjobs/test_calc_job/test_containerized_portable_code.sh diff --git a/.github/config/add-singularity.yaml b/.github/config/add-singularity.yaml new file mode 100644 index 0000000000..6bffe3482f --- /dev/null +++ b/.github/config/add-singularity.yaml @@ -0,0 +1,10 @@ +--- +label: add-singularity +description: Bash run in Docker image through Singularity +default_calc_job_plugin: core.arithmetic.add +computer: localhost +filepath_executable: /bin/sh +image_name: docker://alpine:3 +engine_command: singularity exec --bind $PWD:$PWD {image_name} +prepend_text: ' ' +append_text: ' ' diff --git a/.github/config/add.yaml b/.github/config/add.yaml index d24277501f..f416ec6910 100644 --- a/.github/config/add.yaml +++ b/.github/config/add.yaml @@ -2,7 +2,6 @@ label: add description: add default_calc_job_plugin: core.arithmetic.add -on_computer: true computer: localhost filepath_executable: /bin/bash prepend_text: ' ' diff --git a/.github/config/doubler.yaml b/.github/config/doubler.yaml index 00375f03a9..242598f726 100644 --- a/.github/config/doubler.yaml +++ b/.github/config/doubler.yaml @@ -2,7 +2,6 @@ label: doubler description: doubler default_calc_job_plugin: core.templatereplacer -on_computer: true computer: localhost filepath_executable: PLACEHOLDER_REMOTE_ABS_PATH_DOUBLER prepend_text: ' ' diff --git a/.github/config/localhost.yaml b/.github/config/localhost.yaml index e3187126cd..307a478413 100644 --- a/.github/config/localhost.yaml +++ b/.github/config/localhost.yaml @@ -6,6 +6,7 @@ transport: core.local scheduler: core.direct shebang: '#!/usr/bin/env bash' work_dir: PLACEHOLDER_WORK_DIR +use_double_quotes: true mpirun_command: ' ' mpiprocs_per_machine: 1 prepend_text: ' ' diff --git a/.github/system_tests/test_containerized_code.py b/.github/system_tests/test_containerized_code.py new file mode 100644 index 0000000000..e452ad80bf --- /dev/null +++ b/.github/system_tests/test_containerized_code.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +"""Test running a :class:`~aiida.orm.nodes.data.codes.containerized.ContainerizedCode` code.""" +from aiida import orm +from aiida.engine import run_get_node + + +def test_add_singularity(): + """Test installed containerized code by add plugin""" + builder = orm.load_code('add-singularity@localhost').get_builder() + builder.x = orm.Int(4) + builder.y = orm.Int(6) + builder.metadata.options.resources = {'num_machines': 1, 'num_mpiprocs_per_machine': 1} + + results, node = run_get_node(builder) + + assert node.is_finished_ok + assert 'sum' in results + assert 'remote_folder' in results + assert 'retrieved' in results + assert results['sum'].value == 10 + + +def main(): + test_add_singularity() + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 12db7b6e1a..a3aad8be7b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -44,6 +44,9 @@ jobs: steps: - uses: actions/checkout@v2 + - uses: eWaterCycle/setup-singularity@v7 # for containerized code test + with: + singularity-version: 3.8.7 - name: Cache Python dependencies uses: actions/cache@v1 diff --git a/.github/workflows/setup.sh b/.github/workflows/setup.sh index c6e8f5e144..174e09b598 100755 --- a/.github/workflows/setup.sh +++ b/.github/workflows/setup.sh @@ -23,6 +23,7 @@ verdi computer configure core.local localhost --config "${CONFIG}/localhost-conf verdi computer test localhost verdi code create core.code.installed --non-interactive --config "${CONFIG}/doubler.yaml" verdi code create core.code.installed --non-interactive --config "${CONFIG}/add.yaml" +verdi code create core.code.containerized --non-interactive --config "${CONFIG}/add-singularity.yaml" # set up slurm-ssh computer verdi computer setup --non-interactive --config "${CONFIG}/slurm-ssh.yaml" diff --git a/.github/workflows/tests_nightly.sh b/.github/workflows/tests_nightly.sh index c29b48bc50..10f26f7f15 100755 --- a/.github/workflows/tests_nightly.sh +++ b/.github/workflows/tests_nightly.sh @@ -9,6 +9,7 @@ export PYTHONPATH="${PYTHONPATH}:${SYSTEM_TESTS}:${MODULE_POLISH}" verdi daemon start 4 verdi -p test_aiida run ${SYSTEM_TESTS}/test_daemon.py +verdi -p test_aiida run ${SYSTEM_TESTS}/test_containerized_code.py bash ${SYSTEM_TESTS}/test_polish_workchains.sh verdi daemon stop diff --git a/aiida/orm/__init__.py b/aiida/orm/__init__.py index 1821d65103..e9a26461ed 100644 --- a/aiida/orm/__init__.py +++ b/aiida/orm/__init__.py @@ -49,6 +49,7 @@ 'Comment', 'Computer', 'ComputerEntityLoader', + 'ContainerizedCode', 'DESCENDING', 'Data', 'Dict', diff --git a/aiida/orm/nodes/__init__.py b/aiida/orm/nodes/__init__.py index 5ba5f867fc..3af33b89cc 100644 --- a/aiida/orm/nodes/__init__.py +++ b/aiida/orm/nodes/__init__.py @@ -31,6 +31,7 @@ 'CalculationNode', 'CifData', 'Code', + 'ContainerizedCode', 'Data', 'Dict', 'EnumData', diff --git a/aiida/orm/nodes/data/__init__.py b/aiida/orm/nodes/data/__init__.py index 6d5826eccb..395de5f979 100644 --- a/aiida/orm/nodes/data/__init__.py +++ b/aiida/orm/nodes/data/__init__.py @@ -43,6 +43,7 @@ 'Bool', 'CifData', 'Code', + 'ContainerizedCode', 'Data', 'Dict', 'EnumData', diff --git a/aiida/orm/nodes/data/code/__init__.py b/aiida/orm/nodes/data/code/__init__.py index 200466ecbc..9e85d34d06 100644 --- a/aiida/orm/nodes/data/code/__init__.py +++ b/aiida/orm/nodes/data/code/__init__.py @@ -7,6 +7,7 @@ # pylint: disable=wildcard-import from .abstract import * +from .containerized import * from .installed import * from .legacy import * from .portable import * @@ -14,6 +15,7 @@ __all__ = ( 'AbstractCode', 'Code', + 'ContainerizedCode', 'InstalledCode', 'PortableCode', ) diff --git a/aiida/orm/nodes/data/code/containerized.py b/aiida/orm/nodes/data/code/containerized.py new file mode 100644 index 0000000000..ce268f0f15 --- /dev/null +++ b/aiida/orm/nodes/data/code/containerized.py @@ -0,0 +1,134 @@ +# -*- coding: utf-8 -*- +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +"""Data plugin representing an executable code inside a container. + +The containerized code allows specifying a container image and the executable filepath within that container that is to +be executed when a calculation job is run with this code. +""" +from __future__ import annotations + +import pathlib + +import click + +from aiida.common.lang import type_check + +from .installed import InstalledCode + +__all__ = ('ContainerizedCode',) + + +class ContainerizedCode(InstalledCode): + """Data plugin representing an executable code in container on a remote computer.""" + _KEY_ATTRIBUTE_ENGINE_COMMAND: str = 'engine_command' + _KEY_ATTRIBUTE_IMAGE_NAME: str = 'image_name' + + def __init__(self, engine_command: str, image_name: str, **kwargs): + super().__init__(**kwargs) + self.engine_command = engine_command + self.image_name = image_name + + @property + def filepath_executable(self) -> pathlib.PurePath: + """Return the filepath of the executable that this code represents. + + .. note:: This is overridden from the base class since the path does not have to be absolute. + + :return: The filepath of the executable. + """ + return super().filepath_executable + + @filepath_executable.setter + def filepath_executable(self, value: str) -> None: + """Set the filepath of the executable that this code represents. + + .. note:: This is overridden from the base class since the path does not have to be absolute. + + :param value: The filepath of the executable. + """ + type_check(value, str) + self.base.attributes.set(self._KEY_ATTRIBUTE_FILEPATH_EXECUTABLE, value) + + @property + def engine_command(self) -> str: + """Return the engine command with image as template field of the containerized code + + :return: The engine command of the containerized code + """ + return self.base.attributes.get(self._KEY_ATTRIBUTE_ENGINE_COMMAND) + + @engine_command.setter + def engine_command(self, value: str) -> None: + """Set the engine command of the containerized code + + :param value: The engine command of the containerized code + """ + type_check(value, str) + + if '{image_name}' not in value: + raise ValueError("the '{image_name}' template field should be in engine command.") + + self.base.attributes.set(self._KEY_ATTRIBUTE_ENGINE_COMMAND, value) + + @property + def image_name(self) -> str: + """The image name of container + + :return: The image name of container. + """ + return self.base.attributes.get(self._KEY_ATTRIBUTE_IMAGE_NAME) + + @image_name.setter + def image_name(self, value: str) -> None: + """Set the image name of container + + :param value: The image name of container. + """ + type_check(value, str) + + self.base.attributes.set(self._KEY_ATTRIBUTE_IMAGE_NAME, value) + + def get_prepend_cmdline_params( + self, mpi_args: list[str] | None = None, extra_mpirun_params: list[str] | None = None + ) -> list[str]: + """Return the list of prepend cmdline params for mpi seeting + + :return: list of prepend cmdline parameters.""" + engine_cmdline = self.engine_command.format(image_name=self.image_name) + engine_cmdline_params = engine_cmdline.split() + + return (mpi_args or []) + (extra_mpirun_params or []) + engine_cmdline_params + + @classmethod + def _get_cli_options(cls) -> dict: + """Return the CLI options that would allow to create an instance of this class.""" + options = { + 'engine_command': { + 'required': + True, + 'prompt': + 'Engine command', + 'help': ( + 'The command to run the container. It must contain the placeholder ' + '{image_name} that will be replaced with the `image_name`.' + ), + 'type': + click.STRING, + }, + 'image_name': { + 'required': True, + 'type': click.STRING, + 'prompt': 'Image name', + 'help': 'Name of the image container in which to the run the executable.', + }, + } + options.update(**super()._get_cli_options()) + + return options diff --git a/docs/source/topics/data_types.rst b/docs/source/topics/data_types.rst index 860db0dcad..6048b518f6 100644 --- a/docs/source/topics/data_types.rst +++ b/docs/source/topics/data_types.rst @@ -40,43 +40,46 @@ Below is a list of the core data types already provided with AiiDA, along with t .. table:: :widths: 20 20 45 45 - +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | **Class** | **Entry point** | **Stored in database** | **Stored in repository** | - +==============================================================+========================+===================================================+===================================+ - | :ref:`Int ` | ``core.int`` | The integer value | ``-`` | - +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`Float ` | ``core.float`` | The float value | ``-`` | - +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`Str ` | ``core.str`` | The string | ``-`` | - +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`Bool ` | ``core.bool`` | The boolean value | ``-`` | - +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`List ` | ``core.list`` | The complete list | ``-`` | - +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`Dict ` | ``core.dict`` | The complete dictionary | ``-`` | - +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`EnumData ` | ``core.enum`` | The value, name and the class identifier | ``-`` | - +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`JsonableData ` | ``core.jsonable`` | The JSON data and the class identifier | ``-`` | - +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`ArrayData ` | ``core.array`` | The array names and corresponding shapes | The array data in ``.npy`` format | - +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`XyData ` | ``core.array.xy`` | The array names and corresponding shapes | The array data in ``.npy`` format | - +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`SinglefileData ` | ``core.singlefile`` | The filename | The file | - +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`FolderData ` | ``core.folder`` | ``-`` | All files and folders | - +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`RemoteData ` | ``core.remote`` | The computer and the absolute path to the folder | All files and folders | - +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`AbstractCode ` | ``-`` | Default plugin, append/prepend text | ``-`` | - +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`Code ` | ``core.code`` | The computer and the executable path | All files and folders | - +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`InstalledCode ` | ``core.code.installed``| The computer and the executable path | ``-`` | - +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`PortableCode ` | ``core.code.portable`` | The relative path of the executable | All files and folders of the code | - +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + +-----------------------------------------------------------------------------------------+--------------------------------------+---------------------------------------------------------------------+-----------------------------------+ + | **Class** | **Entry point** | **Stored in database** | **Stored in repository** | + +=========================================================================================+======================================+=====================================================================+===================================+ + | :ref:`Int ` | ``core.int`` | The integer value | ``-`` | + +-----------------------------------------------------------------------------------------+--------------------------------------+---------------------------------------------------------------------+-----------------------------------+ + | :ref:`Float ` | ``core.float`` | The float value | ``-`` | + +-----------------------------------------------------------------------------------------+--------------------------------------+---------------------------------------------------------------------+-----------------------------------+ + | :ref:`Str ` | ``core.str`` | The string | ``-`` | + +-----------------------------------------------------------------------------------------+--------------------------------------+---------------------------------------------------------------------+-----------------------------------+ + | :ref:`Bool ` | ``core.bool`` | The boolean value | ``-`` | + +-----------------------------------------------------------------------------------------+--------------------------------------+---------------------------------------------------------------------+-----------------------------------+ + | :ref:`List ` | ``core.list`` | The complete list | ``-`` | + +-----------------------------------------------------------------------------------------+--------------------------------------+---------------------------------------------------------------------+-----------------------------------+ + | :ref:`Dict ` | ``core.dict`` | The complete dictionary | ``-`` | + +-----------------------------------------------------------------------------------------+--------------------------------------+---------------------------------------------------------------------+-----------------------------------+ + | :ref:`EnumData ` | ``core.enum`` | The value, name and the class identifier | ``-`` | + +-----------------------------------------------------------------------------------------+--------------------------------------+---------------------------------------------------------------------+-----------------------------------+ + | :ref:`JsonableData ` | ``core.jsonable`` | The JSON data and the class identifier | ``-`` | + +-----------------------------------------------------------------------------------------+--------------------------------------+---------------------------------------------------------------------+-----------------------------------+ + | :ref:`ArrayData ` | ``core.array`` | The array names and corresponding shapes | The array data in ``.npy`` format | + +-----------------------------------------------------------------------------------------+--------------------------------------+---------------------------------------------------------------------+-----------------------------------+ + | :ref:`XyData ` | ``core.array.xy`` | The array names and corresponding shapes | The array data in ``.npy`` format | + +-----------------------------------------------------------------------------------------+--------------------------------------+---------------------------------------------------------------------+-----------------------------------+ + | :ref:`SinglefileData ` | ``core.singlefile`` | The filename | The file | + +-----------------------------------------------------------------------------------------+--------------------------------------+---------------------------------------------------------------------+-----------------------------------+ + | :ref:`FolderData ` | ``core.folder`` | ``-`` | All files and folders | + +-----------------------------------------------------------------------------------------+--------------------------------------+---------------------------------------------------------------------+-----------------------------------+ + | :ref:`RemoteData ` | ``core.remote`` | The computer and the absolute path to the folder | All files and folders | + +-----------------------------------------------------------------------------------------+--------------------------------------+---------------------------------------------------------------------+-----------------------------------+ + | :ref:`AbstractCode ` | ``-`` | Default plugin, append/prepend text | ``-`` | + +-----------------------------------------------------------------------------------------+--------------------------------------+---------------------------------------------------------------------+-----------------------------------+ + | :ref:`Code ` | ``core.code`` | The computer and the executable path | All files and folders | + +-----------------------------------------------------------------------------------------+--------------------------------------+---------------------------------------------------------------------+-----------------------------------+ + | :ref:`InstalledCode ` | ``core.code.installed`` | The computer and the executable path | ``-`` | + +-----------------------------------------------------------------------------------------+--------------------------------------+---------------------------------------------------------------------+-----------------------------------+ + | :ref:`PortableCode ` | ``core.code.portable`` | The relative path of the executable | All files and folders of the code | + +-----------------------------------------------------------------------------------------+--------------------------------------+---------------------------------------------------------------------+-----------------------------------+ + | :ref:`ContainerizedCode ` | ``core.code.containerized`` | The computer, the image and the executable path | ``-`` | + +-----------------------------------------------------------------------------------------+--------------------------------------+---------------------------------------------------------------------+-----------------------------------+ + .. _topics:data_types:core:base: @@ -492,8 +495,92 @@ Example of creating an ``PortableCode``: filepath_executable='executable.exe' ) +.. _topics:data_types:core:code:containerized: + +ContainerizedCode +----------------- + +.. versionadded:: 2.1 + +The :class:`~aiida.orm.nodes.data.code.containerized.ContainerizedCode` class allows running an executable within a container image on a target computer. +The data plugin stores the following information in the database: + +* ``image_name``: The name of the container image (e.g., ``docker://alpine:3``). +* ``filepath_executable``: The filepath of the executable within the container (e.g. ``/usr/bin/bash``). +* ``engine_command``: The bash command to invoke the container image (e.g. ``singularity exec --bind $PWD:$PWD {image_name}``). + The exact form of this command will depend on the containerization technology that is used. +* ``computer``: The :class:`~aiida.orm.computers.Computer` on which to run the container. + +.. important:: + + If the ``engine_command`` contains variables (such as in the ``singularity exec --bind $PWD:$PWD {image_name}`` example), it is crucial that the ``Computer`` needs to have the ``use_double_quotes`` setting set to ``True``. + By default, a ``Computer`` will use single quotes to escape command line arguments and so the ``$PWD`` would not be expanded. + If this wasn't defined as such when the ``Computer`` was created, it can be changed through the API: + + .. code-block:: + + computer = load_computer('some-computer') + computer.set_use_double_quotes(True) + +.. note:: + + The container image can be already download and stored on the target computer or can reference an image on a container index, such as Docker Hub. + In the latter case, the first time that the containerized code is run, the image will be automatically pulled from the index, which may take a while if the image is large. + + +.. _topics:data_types:core:code:installed:containerized:setup: + +Setup +^^^^^ + +A ``ContainerizedCode`` can be created through the CLI as well as the API. +The following examples show how to setup running ``bash`` in a base Docker container through Singularity to be run on the ``Computer`` named ``some-computer``: + +.. tab-set:: + + .. tab-item:: CLI + + .. code-block:: console + + verdi code create core.code.containerized \ + --non-interactive \ + --label containerized-code \ + --default-calc-job-plugin core.arithmetic.add \ + --computer some-computer \ + --filepath-executable "/bin/sh" \ + --image-name "docker://alpine:3" \ + --engine-command "singularity exec --bind $PWD:$PWD {image_name}" + + .. tab-item:: API + + .. code-block:: python + + from aiida.orm import ContainerizedCode, load_computer + + code = ContainerizedCode( + computer=load_computer('some-computer') + filepath_executable='/bin/sh' + image_name='docker://alpine:3', + engine_command='singularity exec --bind $PWD:$PWD {image_name}' + ).store() + + +.. _topics:data_types:core:code:installed:containerized:support: + +Supported container technologies +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``ContainerizedCode`` is compatible with a variety of containerization technologies: + +* `Singularity `__ +* `Sarus `__ +* `NVIDIA/enroot `__ +* `Shifter `__ + +Using `Docker `__ directly is currently not supported, since Docker has to run with root permissions and it cannot be launched as a normal MPI program to propagate execution context to the container application. +Support may be added at a later time. + -.. _topics:data_types:materials: Materials science data types ============================ diff --git a/pyproject.toml b/pyproject.toml index a130a81c92..4a185d4bb0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,49 +10,49 @@ authors = [{name = "The AiiDA team", email = "developers@aiida.net"}] readme = "README.md" license = {file = "LICENSE.txt"} classifiers = [ - "Development Status :: 5 - Production/Stable", - "Framework :: AiiDA", - "License :: OSI Approved :: MIT License", - "Operating System :: POSIX :: Linux", - "Operating System :: MacOS :: MacOS X", - "Programming Language :: Python", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Topic :: Scientific/Engineering" + "Development Status :: 5 - Production/Stable", + "Framework :: AiiDA", + "License :: OSI Approved :: MIT License", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS :: MacOS X", + "Programming Language :: Python", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Topic :: Scientific/Engineering" ] keywords = ["aiida", "workflows"] requires-python = ">=3.8" dependencies = [ - "alembic~=1.2", - "archive-path~=0.4.1", - "aio-pika~=6.6", - "circus~=0.17.1", - "click-config-file~=0.6.0", - "click-spinner~=0.1.8", - "click~=8.1", - "disk-objectstore~=0.6.0", - "graphviz~=0.13", - "ipython~=7.20", - "jinja2~=3.0", - "jsonschema~=3.0", - "kiwipy[rmq]~=0.7.5", - "importlib-metadata~=4.3", - "numpy~=1.19", - "pamqp~=2.3", - "paramiko~=2.7,>=2.7.2", - "plumpy~=0.21.0", - "pgsu~=0.2.1", - "psutil~=5.6", - "psycopg2-binary~=2.8", - "pytz~=2021.1", - "pyyaml~=5.4", - "sqlalchemy~=1.4.22", - "tabulate~=0.8.5", - "tqdm~=4.45", - "upf_to_json~=0.9.2", - "werkzeug<2.2", - "wrapt~=1.11" + "alembic~=1.2", + "archive-path~=0.4.1", + "aio-pika~=6.6", + "circus~=0.17.1", + "click-config-file~=0.6.0", + "click-spinner~=0.1.8", + "click~=8.1", + "disk-objectstore~=0.6.0", + "graphviz~=0.13", + "ipython~=7.20", + "jinja2~=3.0", + "jsonschema~=3.0", + "kiwipy[rmq]~=0.7.5", + "importlib-metadata~=4.3", + "numpy~=1.19", + "pamqp~=2.3", + "paramiko~=2.7,>=2.7.2", + "plumpy~=0.21.0", + "pgsu~=0.2.1", + "psutil~=5.6", + "psycopg2-binary~=2.8", + "pytz~=2021.1", + "pyyaml~=5.4", + "sqlalchemy~=1.4.22", + "tabulate~=0.8.5", + "tqdm~=4.45", + "upf_to_json~=0.9.2", + "werkzeug<2.2", + "wrapt~=1.11" ] [project.urls] @@ -171,6 +171,7 @@ runaiida = "aiida.cmdline.commands.cmd_run:run" "core.code" = "aiida.orm.nodes.data.code.legacy:Code" "core.code.portable" = "aiida.orm.nodes.data.code.portable:PortableCode" "core.code.installed" = "aiida.orm.nodes.data.code.installed:InstalledCode" +"core.code.containerized" = "aiida.orm.nodes.data.code.containerized:ContainerizedCode" "core.dict" = "aiida.orm.nodes.data.dict:Dict" "core.enum" = "aiida.orm.nodes.data.enum:EnumData" "core.float" = "aiida.orm.nodes.data.float:Float" diff --git a/tests/cmdline/commands/test_code.py b/tests/cmdline/commands/test_code.py index 35e74a159d..2706113373 100644 --- a/tests/cmdline/commands/test_code.py +++ b/tests/cmdline/commands/test_code.py @@ -413,19 +413,34 @@ def command_options(request, aiida_localhost, tmp_path): """Return tuple of list of options and entry point.""" options = [request.param, '-n', '--label', str(uuid.uuid4())] - if request.param == 'core.code.installed': + if 'installed' in request.param: options.extend(['--computer', str(aiida_localhost.pk), '--filepath-executable', '/usr/bin/bash']) - if request.param == 'core.code.portable': + if 'portable' in request.param: filepath_executable = 'bash' (tmp_path / filepath_executable).touch() options.extend(['--filepath-executable', filepath_executable, '--filepath-files', tmp_path]) + if 'containerized' in request.param: + engine_command = 'singularity exec --bind $PWD:$PWD {image_name}' + image_name = 'ubuntu' + options.extend([ + '--computer', + str(aiida_localhost.pk), '--filepath-executable', '/usr/bin/bash', '--engine-command', engine_command, + '--image-name', image_name + ]) + return options, request.param @pytest.mark.usefixtures('aiida_profile_clean') -@pytest.mark.parametrize('command_options', ('core.code.installed', 'core.code.portable'), indirect=True) +@pytest.mark.parametrize( + 'command_options', ( + 'core.code.installed', + 'core.code.portable', + 'core.code.containerized', + ), indirect=True +) @pytest.mark.parametrize('non_interactive_editor', ('sleep 1; vim -cwq',), indirect=True) def test_code_create(run_cli_command, command_options, non_interactive_editor): """Test the ``verdi code create`` command.""" diff --git a/tests/engine/processes/calcjobs/test_calc_job.py b/tests/engine/processes/calcjobs/test_calc_job.py index b25d9fd729..c6e35d38b7 100644 --- a/tests/engine/processes/calcjobs/test_calc_job.py +++ b/tests/engine/processes/calcjobs/test_calc_job.py @@ -14,6 +14,7 @@ import io import json import os +import pathlib import tempfile from unittest.mock import patch @@ -226,6 +227,42 @@ def test_code_double_quotes(aiida_localhost, file_regression, code_use_double_qu file_regression.check(content, extension='.sh') +@pytest.mark.requires_rmq +@pytest.mark.usefixtures('clear_database_before_test', 'chdir_tmp_path') +def test_containerized_installed_code(file_regression, aiida_localhost): + """test run container code""" + aiida_localhost.set_use_double_quotes(True) + engine_command = """singularity exec --bind $PWD:$PWD {image_name}""" + containerized_code = orm.ContainerizedCode( + default_calc_job_plugin='core.arithmetic.add', + filepath_executable='/bin/bash', + engine_command=engine_command, + image_name='ubuntu', + computer=aiida_localhost, + ).store() + + inputs = { + 'code': containerized_code, + 'metadata': { + 'dry_run': True, + 'options': { + 'resources': { + 'num_machines': 1, + 'num_mpiprocs_per_machine': 1 + }, + 'withmpi': False, + } + } + } + + _, node = launch.run_get_node(DummyCalcJob, **inputs) + folder_name = node.dry_run_info['folder'] + submit_script_filename = node.get_option('submit_script_filename') + content = (pathlib.Path(folder_name) / submit_script_filename).read_bytes().decode('utf-8') + + file_regression.check(content, extension='.sh') + + @pytest.mark.requires_rmq @pytest.mark.usefixtures('aiida_profile_clean', 'chdir_tmp_path') @pytest.mark.parametrize('calcjob_withmpi', [True, False]) @@ -260,7 +297,6 @@ def test_multi_codes_run_withmpi(aiida_local_code_factory, file_regression, calc @pytest.mark.usefixtures('clear_database_before_test', 'chdir_tmp_path') def test_portable_code(tmp_path, aiida_localhost): """test run container code""" - import pathlib (tmp_path / 'bash').write_bytes(b'bash implementation') subdir = tmp_path / 'sub' subdir.mkdir() diff --git a/tests/engine/processes/calcjobs/test_calc_job/test_containerized_installed_code.sh b/tests/engine/processes/calcjobs/test_calc_job/test_containerized_installed_code.sh new file mode 100644 index 0000000000..38eb248b6a --- /dev/null +++ b/tests/engine/processes/calcjobs/test_calc_job/test_containerized_installed_code.sh @@ -0,0 +1,6 @@ +#!/bin/bash +exec > _scheduler-stdout.txt +exec 2> _scheduler-stderr.txt + + +"singularity" "exec" "--bind" "$PWD:$PWD" "ubuntu" '/bin/bash' '--version' '-c' < "aiida.in" > "aiida.out" 2> "aiida.err" diff --git a/tests/engine/processes/calcjobs/test_calc_job/test_containerized_portable_code.sh b/tests/engine/processes/calcjobs/test_calc_job/test_containerized_portable_code.sh new file mode 100644 index 0000000000..a5527c40d0 --- /dev/null +++ b/tests/engine/processes/calcjobs/test_calc_job/test_containerized_portable_code.sh @@ -0,0 +1,6 @@ +#!/bin/bash +exec > _scheduler-stdout.txt +exec 2> _scheduler-stderr.txt + + +"singularity" "exec" "--bind" "$PWD:$PWD" "ubuntu" 'bash' < "aiida.in" > "aiida.out" From 0620b73fca3e7d4c1c93a78eba5281b170d2bb58 Mon Sep 17 00:00:00 2001 From: Sebastiaan Huber Date: Tue, 18 Oct 2022 16:29:04 +0200 Subject: [PATCH 2/3] Address PR review comments --- docs/source/topics/data_types.rst | 72 ++++++++++++++++--- .../processes/calcjobs/test_calc_job.py | 40 ++++++++++- ...led_code.sh => test_containerized_code.sh} | 0 .../test_containerized_code_withmpi_true.sh | 6 ++ .../test_containerized_portable_code.sh | 6 -- 5 files changed, 105 insertions(+), 19 deletions(-) rename tests/engine/processes/calcjobs/test_calc_job/{test_containerized_installed_code.sh => test_containerized_code.sh} (100%) create mode 100644 tests/engine/processes/calcjobs/test_calc_job/test_containerized_code_withmpi_true.sh delete mode 100644 tests/engine/processes/calcjobs/test_calc_job/test_containerized_portable_code.sh diff --git a/docs/source/topics/data_types.rst b/docs/source/topics/data_types.rst index 6048b518f6..558646367b 100644 --- a/docs/source/topics/data_types.rst +++ b/docs/source/topics/data_types.rst @@ -505,12 +505,17 @@ ContainerizedCode The :class:`~aiida.orm.nodes.data.code.containerized.ContainerizedCode` class allows running an executable within a container image on a target computer. The data plugin stores the following information in the database: -* ``image_name``: The name of the container image (e.g., ``docker://alpine:3``). +* ``image_name``: The name of the container image (e.g., a URI like ``docker://alpine:3`` or an absolute file path like ``/path/to/image.sif``). * ``filepath_executable``: The filepath of the executable within the container (e.g. ``/usr/bin/bash``). * ``engine_command``: The bash command to invoke the container image (e.g. ``singularity exec --bind $PWD:$PWD {image_name}``). The exact form of this command will depend on the containerization technology that is used. * ``computer``: The :class:`~aiida.orm.computers.Computer` on which to run the container. +.. note:: + + If the container image is not yet present on the target computer, most container engines will pull the image from the registry at first use. + This can take a while if the image is large. + .. important:: If the ``engine_command`` contains variables (such as in the ``singularity exec --bind $PWD:$PWD {image_name}`` example), it is crucial that the ``Computer`` needs to have the ``use_double_quotes`` setting set to ``True``. @@ -522,11 +527,6 @@ The data plugin stores the following information in the database: computer = load_computer('some-computer') computer.set_use_double_quotes(True) -.. note:: - - The container image can be already download and stored on the target computer or can reference an image on a container index, such as Docker Hub. - In the latter case, the first time that the containerized code is run, the image will be automatically pulled from the index, which may take a while if the image is large. - .. _topics:data_types:core:code:installed:containerized:setup: @@ -564,6 +564,38 @@ The following examples show how to setup running ``bash`` in a base Docker conta engine_command='singularity exec --bind $PWD:$PWD {image_name}' ).store() +Please refer to the section on :ref:`supported container technologies ` for an overview and specific setup instructions for each containerization solution. + +.. _topics:data_types:core:code:installed:containerized:run: + +Run +^^^ + +A ``ContainerizedCode`` is used to launch a calculation just like any other code. +If a default calculation job plugin is defined, a process builder can be obtained with ``get_builder``: + +.. code-block:: python + + from aiida.engine import submit + from aiida.orm import load_code + + code = load_code('containerized-code') + builder = code.get_builder() + # Define the rest of the inputs + submit(builder) + +.. important:: + + If a containerized code is used for a calculation that sets the :ref:`metadata option ` ``withmpi`` to ``True``, the MPI command line arguments are placed in front of the container runtime. + For example, when running Singularity with ``metadata.options.withmpi = True``, the runline in the submission script will be written as: + + .. code-block:: bash + + "mpirun" "-np" "1" "singularity" "exec" "--bind" "$PWD:$PWD" "ubuntu" '/bin/bash' '--version' '-c' < "aiida.in" > "aiida.out" 2> "aiida.err" + + This means that the containerization program is launched as a normal MPI program, and so it needs to support forwarding the execution context to the container application. + It is currently not possible to have MPI invoked inside the container runtime. + .. _topics:data_types:core:code:installed:containerized:support: @@ -572,12 +604,30 @@ Supported container technologies The ``ContainerizedCode`` is compatible with a variety of containerization technologies: -* `Singularity `__ -* `Sarus `__ -* `NVIDIA/enroot `__ -* `Shifter `__ +.. tab-set:: + + .. tab-item:: Singularity + + To use `Singularity `__ use the following ``engine_command`` when setting up the code: + + .. code-block:: console + + singularity exec --bind $PWD:$PWD {image_name} + + .. tab-item:: Sarus + + To use `Sarus `__ use the following ``engine_command`` when setting up the code: + + .. code-block:: console + + sarus run {image_name} + + +Using `Docker `__ directly is currently not supported because: + +* Docker needs to be run as the root user and the files created in the working directory inside the container will be owned by root, which prevents AiiDA from deleting those files after execution. +* Docker cannot be launched as a normal MPI program to propagate execution context to the container application. -Using `Docker `__ directly is currently not supported, since Docker has to run with root permissions and it cannot be launched as a normal MPI program to propagate execution context to the container application. Support may be added at a later time. diff --git a/tests/engine/processes/calcjobs/test_calc_job.py b/tests/engine/processes/calcjobs/test_calc_job.py index c6e35d38b7..7ec6ec1b00 100644 --- a/tests/engine/processes/calcjobs/test_calc_job.py +++ b/tests/engine/processes/calcjobs/test_calc_job.py @@ -229,8 +229,8 @@ def test_code_double_quotes(aiida_localhost, file_regression, code_use_double_qu @pytest.mark.requires_rmq @pytest.mark.usefixtures('clear_database_before_test', 'chdir_tmp_path') -def test_containerized_installed_code(file_regression, aiida_localhost): - """test run container code""" +def test_containerized_code(file_regression, aiida_localhost): + """Test the :class:`~aiida.orm.nodes.data.code.containerized.ContainerizedCode`.""" aiida_localhost.set_use_double_quotes(True) engine_command = """singularity exec --bind $PWD:$PWD {image_name}""" containerized_code = orm.ContainerizedCode( @@ -263,6 +263,42 @@ def test_containerized_installed_code(file_regression, aiida_localhost): file_regression.check(content, extension='.sh') +@pytest.mark.requires_rmq +@pytest.mark.usefixtures('clear_database_before_test', 'chdir_tmp_path') +def test_containerized_code_withmpi_true(file_regression, aiida_localhost): + """Test the :class:`~aiida.orm.nodes.data.code.containerized.ContainerizedCode` with ``withmpi=True``.""" + aiida_localhost.set_use_double_quotes(True) + engine_command = """singularity exec --bind $PWD:$PWD {image_name}""" + containerized_code = orm.ContainerizedCode( + default_calc_job_plugin='core.arithmetic.add', + filepath_executable='/bin/bash', + engine_command=engine_command, + image_name='ubuntu', + computer=aiida_localhost, + ).store() + + inputs = { + 'code': containerized_code, + 'metadata': { + 'dry_run': True, + 'options': { + 'resources': { + 'num_machines': 1, + 'num_mpiprocs_per_machine': 1 + }, + 'withmpi': True, + } + } + } + + _, node = launch.run_get_node(DummyCalcJob, **inputs) + folder_name = node.dry_run_info['folder'] + submit_script_filename = node.get_option('submit_script_filename') + content = (pathlib.Path(folder_name) / submit_script_filename).read_bytes().decode('utf-8') + + file_regression.check(content, extension='.sh') + + @pytest.mark.requires_rmq @pytest.mark.usefixtures('aiida_profile_clean', 'chdir_tmp_path') @pytest.mark.parametrize('calcjob_withmpi', [True, False]) diff --git a/tests/engine/processes/calcjobs/test_calc_job/test_containerized_installed_code.sh b/tests/engine/processes/calcjobs/test_calc_job/test_containerized_code.sh similarity index 100% rename from tests/engine/processes/calcjobs/test_calc_job/test_containerized_installed_code.sh rename to tests/engine/processes/calcjobs/test_calc_job/test_containerized_code.sh diff --git a/tests/engine/processes/calcjobs/test_calc_job/test_containerized_code_withmpi_true.sh b/tests/engine/processes/calcjobs/test_calc_job/test_containerized_code_withmpi_true.sh new file mode 100644 index 0000000000..0c1d1f2100 --- /dev/null +++ b/tests/engine/processes/calcjobs/test_calc_job/test_containerized_code_withmpi_true.sh @@ -0,0 +1,6 @@ +#!/bin/bash +exec > _scheduler-stdout.txt +exec 2> _scheduler-stderr.txt + + +"mpirun" "-np" "1" "singularity" "exec" "--bind" "$PWD:$PWD" "ubuntu" '/bin/bash' '--version' '-c' < "aiida.in" > "aiida.out" 2> "aiida.err" diff --git a/tests/engine/processes/calcjobs/test_calc_job/test_containerized_portable_code.sh b/tests/engine/processes/calcjobs/test_calc_job/test_containerized_portable_code.sh deleted file mode 100644 index a5527c40d0..0000000000 --- a/tests/engine/processes/calcjobs/test_calc_job/test_containerized_portable_code.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -exec > _scheduler-stdout.txt -exec 2> _scheduler-stderr.txt - - -"singularity" "exec" "--bind" "$PWD:$PWD" "ubuntu" 'bash' < "aiida.in" > "aiida.out" From c3298dd7e82b85db6628cfa32efbc3f52ecdb5e7 Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Tue, 18 Oct 2022 17:34:26 +0200 Subject: [PATCH 3/3] after review: rephrase docker exclu and and sarus engine command --- docs/source/topics/data_types.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/topics/data_types.rst b/docs/source/topics/data_types.rst index 558646367b..9e777b2957 100644 --- a/docs/source/topics/data_types.rst +++ b/docs/source/topics/data_types.rst @@ -620,12 +620,12 @@ The ``ContainerizedCode`` is compatible with a variety of containerization techn .. code-block:: console - sarus run {image_name} + sarus run --mount=src=$PWD,dst=/workdir,type=bind --workdir=/workdir {image_name} Using `Docker `__ directly is currently not supported because: -* Docker needs to be run as the root user and the files created in the working directory inside the container will be owned by root, which prevents AiiDA from deleting those files after execution. +* The Docker daemon always runs as the root user and the files created in the working directory inside the container will usually be owned by root if uid is not specified in the image, which prevents AiiDA from deleting those files after execution. * Docker cannot be launched as a normal MPI program to propagate execution context to the container application. Support may be added at a later time.