DOC and TEST: containerized code how to set and how to use

unkcpz · unkcpz · commit e47766597bb6 · 2022-08-31T12:21:09.000+02:00
pytest fix

Add more docs in data_types

Use localhost to test add-docker container
diff --git a/.github/config/add.yaml b/.github/config/add.yaml
@@ -1,9 +1,9 @@
 ---
 label: add
 description: add
-default_calc_job_plugin: core.arithmetic.add
+input_plugin: core.arithmetic.add
 on_computer: true
 computer: localhost
-filepath_executable: /bin/bash
+remote_abs_path: /bin/bash
 prepend_text: ' '
 append_text: ' '
diff --git a/.github/config/doubler.yaml b/.github/config/doubler.yaml
@@ -1,9 +1,9 @@
 ---
 label: doubler
 description: doubler
-default_calc_job_plugin: core.templatereplacer
+input_plugin: core.templatereplacer
 on_computer: true
 computer: localhost
-filepath_executable: PLACEHOLDER_REMOTE_ABS_PATH_DOUBLER
+remote_abs_path: PLACEHOLDER_REMOTE_ABS_PATH_DOUBLER
 prepend_text: ' '
 append_text: ' '
diff --git a/.github/system_tests/test_containerized_code.py b/.github/system_tests/test_containerized_code.py
@@ -1,9 +1,18 @@
-import os
+# -*- coding: utf-8 -*-
+###########################################################################
+# Copyright (c), The AiiDA team. All rights reserved.                     #
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
+"""Test run on containrized code."""
 from aiida import orm
 from aiida.engine import run_get_node
 from aiida.plugins import CalculationFactory
 
-ArithmeticAddCalculation = CalculationFactory('core.arithmetic.add') 
+ArithmeticAddCalculation = CalculationFactory('core.arithmetic.add')
 
 inputs = {
     'code': orm.load_code('add-docker@localhost'),
diff --git a/.github/workflows/setup.sh b/.github/workflows/setup.sh
@@ -21,8 +21,8 @@ verdi setup --non-interactive --config "${CONFIG}/profile.yaml"
 verdi computer setup --non-interactive --config "${CONFIG}/localhost.yaml"
 verdi computer configure core.local localhost --config "${CONFIG}/localhost-config.yaml"
 verdi computer test localhost
-verdi code create core.code.installed --non-interactive --config "${CONFIG}/doubler.yaml"
-verdi code create core.code.installed --non-interactive --config "${CONFIG}/add.yaml"
+verdi code setup --non-interactive --config "${CONFIG}/doubler.yaml"
+verdi code setup --non-interactive --config "${CONFIG}/add.yaml"
 verdi code create core.code.installed.containerized --non-interactive --config "${CONFIG}/add-docker.yaml"
 
 # set up slurm-ssh computer
diff --git a/.github/workflows/tests_nightly.sh b/.github/workflows/tests_nightly.sh
@@ -9,7 +9,6 @@ export PYTHONPATH="${PYTHONPATH}:${SYSTEM_TESTS}:${MODULE_POLISH}"
 
 verdi daemon start 4
 verdi -p test_aiida run ${SYSTEM_TESTS}/test_daemon.py
-docker version | echo
 verdi -p test_aiida run ${SYSTEM_TESTS}/test_containerized_code.py
 bash ${SYSTEM_TESTS}/test_polish_workchains.sh
 verdi daemon stop
diff --git a/aiida/engine/daemon/execmanager.py b/aiida/engine/daemon/execmanager.py
@@ -29,7 +29,6 @@
 from aiida.common.links import LinkType
 from aiida.manage.configuration import get_config_option
 from aiida.orm import CalcJobNode, Code, FolderData, Node, PortableCode, RemoteData, load_node
-from aiida.orm.nodes.data.code.containerized import PortableContainerizedCode
 from aiida.orm.utils.log import get_dblogger_extra
 from aiida.repository.common import FileType
 from aiida.schedulers.datastructures import JobState
diff --git a/aiida/engine/processes/calcjobs/calcjob.py b/aiida/engine/processes/calcjobs/calcjob.py
@@ -24,7 +24,7 @@
 from aiida.common.folders import Folder
 from aiida.common.lang import classproperty, override
 from aiida.common.links import LinkType
-from aiida.orm.nodes.data.code.containerized import Containerized, PortableContainerizedCode
+from aiida.orm.nodes.data.code.containerized import Containerized
 from aiida.orm.utils.loaders import load_code
 
 from ..exit_code import ExitCode
@@ -617,15 +617,7 @@ def presubmit(self, folder: Folder) -> CalcInfo:
         from aiida.common.datastructures import CodeInfo, CodeRunMode
         from aiida.common.exceptions import InputValidationError, InvalidOperation, PluginInternalError, ValidationError
         from aiida.common.utils import validate_list_of_string_tuples
-        from aiida.orm import (
-            AbstractCode,
-            Code,
-            Computer,
-            InstalledCode,
-            InstalledContainerizedCode,
-            PortableCode,
-            load_node,
-        )
+        from aiida.orm import AbstractCode, Computer, PortableCode
         from aiida.schedulers.datastructures import JobTemplate, JobTemplateCodeInfo
 
         inputs = self.node.base.links.get_incoming(link_type=LinkType.INPUT_CALC)
diff --git a/aiida/orm/nodes/data/code/containerized.py b/aiida/orm/nodes/data/code/containerized.py
@@ -112,11 +112,16 @@ def _get_cli_options(cls) -> dict:
         """Return the CLI options that would allow to create an instance of this class."""
         options = {
             'engine_command': {
-                'required': True,
-                'prompt': 'Engine command',
-                'help':
-                'The command to run the container. It must contain the placeholder {image} that will be replaced with the `image_name`.',
-                'type': click.STRING,
+                'required':
+                True,
+                'prompt':
+                'Engine command',
+                'help': (
+                    'The command to run the container. It must contain the placeholder '
+                    '{image} that will be replaced with the `image_name`.'
+                ),
+                'type':
+                click.STRING,
             },
             'image': {
                 'required': True,
diff --git a/aiida/schedulers/scheduler.py b/aiida/schedulers/scheduler.py
@@ -214,6 +214,7 @@ def _get_run_line(self, codes_info, codes_run_mode):
             to launch the multiple codes.
         :return: string with format: [executable] [args] {[ < stdin ]} {[ < stdout ]} {[2>&1 | 2> stderr]}
         """
+        # pylint: disable=too-many-locals
         from aiida.common.datastructures import CodeRunMode
 
         list_of_runlines = []
diff --git a/docs/source/howto/index.rst b/docs/source/howto/index.rst
@@ -8,6 +8,7 @@ How-To Guides
    interact
    plugins_install
    run_codes
+   run_containerized_codes
    run_workflows
    ssh
    plugin_codes
diff --git a/docs/source/howto/run_containerized_codes.rst b/docs/source/howto/run_containerized_codes.rst
@@ -0,0 +1,159 @@
+.. _how-to:run-containerized_codes:
+
+****************************************
+How to setup and run containerized codes
+****************************************
+
+This how-to walks you through the steps of setting up a (possibly remote) compute resource, setting up a containerized code on that computer, and submitting a calculation through AiiDA (with more details supplement to :ref:`introductory tutorial <tutorial:basic:calcjob>`).
+
+.. important::
+
+    Since running code in then container require mapping the current directory where the input files locate to the working directory in the container, the current directory is specified in the job script as ``$PWD``.
+    The computer setup to run containerized code MUST set ``use_double_quotes`` to ``True`` to escape the command line parameters using double quotes.
+
+There are two type of codes the ``installed`` and ``portable``, which are all supported to be run in the container.
+
+Docker containerized code
+=========================
+
+Running installed code within container
+---------------------------------------
+
+The typical code setup for for running code installed docker container is shown below.
+
+    .. code-block::yaml
+
+        ---
+        label: add-docker
+        description: add docker
+        default_calc_job_plugin: core.arithmetic.add
+        on_computer: true
+        computer: localhost
+        filepath_executable: /bin/bash
+        image: ubuntu
+        engine_command: docker run -v $PWD:/workdir:rw -w /workdir {image} sh -c
+        escape_exec_line: true
+        prepend_text: ' '
+        append_text: ' '
+
+The option ``escape_exec_line`` is mandatory for running commands in docker which need to have executable command in the quotes after ``sh -c`` of engine command.
+This opiton will escape the ``cmdline_params`` along with the redirect parameters in the quotes so the whole command is recogonized and run inside the docker container.
+
+Then this code can be used the same way as regular setup code.
+There is no need to explicitly imstall or compile the code on the machine, but only require that the executable is in the container shipped with docker image.
+For the first time running the docker containerized code, the image will pulled from dockerhub automatically, which will cost sometime if the image is large.
+
+.. important::
+
+    One issue with running code in docker container is the files output is owned by ``root``, which lead to the remote folder can not be cleaned by current user.
+
+Running portable code within container
+--------------------------------------
+
+Using container as the environment to run the portable script code without isolating environment and installing libraries.
+It is common to have some codes which are scripts for example the python scripts that need to run with libraries dependencies installed prehead.
+It is not easy to isolate the environment and install the dependencies everytime before setup the code for AiiDA.
+With portable code option, the container can be regard as the environment with dependencies installed.
+The user just need to setup the script code as portable code and run it inside the container where has all dependencies fullfilled.
+Therefore, the code can be run on whatever computer that has docker istalled, which reduce the burden to sophisticately configure the environment and dependencies in computer.
+
+The code setup config example is:
+
+    .. code-block:: yaml
+
+        ---
+        label: "docker-python-add"
+        description: "doing python add"
+        input_plugin: "core.arithmetic.add"
+        image: "python:3.9.12-buster"
+        engine_command: "docker run -v $PWD:/workdir:rw -w /workdir {image} sh -c"
+        escape_exec_line: true
+        filepath_executable: "./eval_sh.py"
+        filepath_files: "<Filepath to directory containing code files>"
+        prepend_text: " "
+        append_text: " "
+
+where the executable ``eval_sh.py`` is a dummy python script that execute ``bash < aiida.in > aiida.out``.
+It can be a much complex script with many different libraries imported such as ``numpy``, ``scipy``, only if the container image has this libraries installed.
+
+    .. code-block:: python
+
+        #!/usr/bin/env python
+        import os
+
+        if __name__ == '__main__':
+            inputfile = 'aiida.in'
+            outputfile = 'aiida.out'
+
+            with open(inputfile, 'r') as f:
+                bashCommand = '/bin/bash < aiida.in > aiida.out'
+                os.system(bashCommand)
+
+To run this containerized portable code, you need to specify the computer where has docker installed in ``CalcJob`` metadata as:
+
+    .. code-block:: python
+
+        from aiida import orm
+        from aiida.engine import run_get_node
+        from aiida.plugins import CalculationFactory
+
+        ArithmeticAddCalculation = CalculationFactory('core.arithmetic.add')
+
+        inputs = {
+            'code': orm.load_code('docker-python-add'),
+            'x': orm.Int(4),
+            'y': orm.Int(6),
+            'metadata': {
+                'computer': orm.load_computer('localhost'), # localhost has docker installed
+                'options': {
+                    'resources': {
+                        'num_machines': 1,
+                        'num_mpiprocs_per_machine': 1
+                    }
+                }
+            }
+        }
+
+        _, node = run_get_node(ArithmeticAddCalculation, **inputs)
+
+Just specify the computer and user are free from worrying about the dependencies on the computer.
+
+Containerized code run by Singularity and Sarus
+===============================================
+
+Sarus and Singularity are softwares to run Linux containers on High Performance Computing environments.
+Their development have been driven by the specific requirements of HPC system, while leveraging open standards and technologies to encourage vendor and community involement.
+
+The simulation codes are shipping inside the container which can be hardware agnostic.
+The images are built by users to fit the deployment of a specific application, and spawing of isolated environments (containers) when running.
+Users can install any code to remote cluster by simply pull the image.
+
+The Sarus and Singularity share the same logic of using the the containerized technology.
+The only difference is the details of the command to launch the container and call/running the softwares for inside, which can be specified by ``engine_command`` when setup the code.
+
+The typical code setup for for running Sarus containerized code is shown below.
+
+    .. code-block::yaml
+
+        ---
+        label: sarus-qe
+        description: The QuantumESPRESSO in the Sarus container.
+        default_calc_job_plugin: <calc_job_plugin>
+        on_computer: true
+        computer: <remote computer with Sarus available>
+        filepath_executable: <path to executable inside the container>
+        image: <image will replace the template of image in engine_command>
+        engine_command: sarus run --mount=src=$PWD,dst=/workdir,type=bind --workdir=/rundir {image}
+        escape_exec_line: false
+        prepend_text: ' '
+        append_text: ' '
+
+For the Singularity the ``engine_command`` is ``singularity exec --bind $PWD:$PWD``
+
+Then this code can be used the same way as regular setup code.
+There is no need to explicitly imstall or compile the code on the machine, but only require that the executable is in the container shipped with docker image.
+For the first time running the docker containerized code, the image will pulled from dockerhub automatically, which will cost sometime if the image is large.
+
+.. important::
+
+    The image need to be exist on the remote computer before you can use them. To install the containerized code just need to pull the image if the it is prepared.
diff --git a/docs/source/topics/data_types.rst b/docs/source/topics/data_types.rst
@@ -422,6 +422,8 @@ There are currently three implementations of this abstract class:
  * :class:`~aiida.orm.nodes.data.code.legacy.Code` (see :ref:`Code <topics:data_types:core:code:legacy>`)
  * :class:`~aiida.orm.nodes.data.code.installed.InstalledCode` (see :ref:`InstalledCode <topics:data_types:core:code:installed>`)
  * :class:`~aiida.orm.nodes.data.code.portable.PortableCode` (see :ref:`PortableCode <topics:data_types:core:code:portable>`)
+ * :class:`~aiida.orm.nodes.data.code.containerized.InstalledContainerizedCode` (see :ref:`ContainerizedCode <topics:data_types:core:code:containerized>`)
+ * :class:`~aiida.orm.nodes.data.code.containerized.PortableContainerizedCode` (see :ref:`ContainerzedCode <topics:data_types:core:code:containerized>`)
 
 
 .. _topics:data_types:core:code:legacy:
@@ -492,6 +494,51 @@ Example of creating an ``PortableCode``:
         filepath_executable='executable.exe'
     )
 
+.. _topics:data_types:core:code:containerized:
+
+Containerized Code
+------------------
+
+.. versionadded:: 2.1
+
+A containerzed code is the code shipped with container.
+It can be either the :class:`~aiida.orm.nodes.data.code.containerized.PortableContainerizedCode` class or :class:`~aiida.orm.nodes.data.code.containerized.InstalledContainerizedCode` class.
+Which are implementations of the :class:`~aiida.orm.nodes.data.code.abstract.AbstractCode` class with :class:`~aiida.orm.nodes.data.code.portable.PortableCode` or :class:`~aiida.orm.nodes.data.code.installed.InstalledCode` respectively.
+
+Example of creating an ``InstalledContainerizedCode``:
+
+.. code:: python
+
+    from pathlib import Path
+    from aiida.orm import InstalledContainerizedCode
+    code = InstalledContainerizedCode(
+        label='some-label',
+        computer=load_computer('localhost'),
+        filepath_executable='/usr/bin/bash'
+        engine_command='engine_command',
+        image='imagename',
+        escape_exec_line=False,
+    )
+
+
+Example of creating an ``PortableContainerizedCode``:
+
+.. code:: python
+
+    from pathlib import Path
+    from aiida.orm import PortableContainerizedCode
+    code = PortableContainerizedCode(
+        label='some-label',
+        filepath_files=Path('/some/path/code'),
+        filepath_executable='executable.exe',
+        engine_command='engine_command',
+        image='imagename',
+        escape_exec_line=True,
+    )
+
+.. note::
+
+    The option ``escape_exec_line`` is mandatory for running commands in docker which need to have executable command in the quotes after ``sh -c`` of engine command.
 
 .. _topics:data_types:materials:
 
diff --git a/tests/cmdline/commands/test_code.py b/tests/cmdline/commands/test_code.py
@@ -10,7 +10,6 @@
 # pylint: disable=unused-argument,redefined-outer-name
 """Tests for the 'verdi code' command."""
 import io
-from ipaddress import ip_address
 import os
 import pathlib
 import tempfile
