From 82c45bb4e467b0e3b75d390f4ed353eb149bffb5 Mon Sep 17 00:00:00 2001 From: Sebastiaan Huber Date: Sat, 30 Apr 2022 20:02:56 +0200 Subject: [PATCH 1/7] Add the `InstalledCode` and `PortableCode` data plugins Historically, the `Code` data plugin came in two flavors: codes that represent an executable on an associated `Computer`, and codes that were included in the repository of the node instance itself. For the latter, when used in a `CalcJob` its files would be uploaded to the remote working directory. The use of a single class for these two different types of entities has caused quite a bit of confusion, not only for end-users, but also for developers as it leads to convoluted code. Here, we add two new data plugins, `InstalledCode` and `PortableCode` that are designed to implement the exact same behavior, but do so in a more user-friendly and maintainable way by having them as separate classes. The both subclass from `AbstractCode` which provides the functionality that is shared between them. For backwards compatibility, the original `Code` implementation is kept, but it is moved to the `aiida.orm.nodes.data.code.legacy` module. The idea is that this class is deprecated and users are encouraged to start using the new classes. In a future version, the `Code` class will be decommissioned and an automatic migration will be included to migrate its existing entries to a `InstalledCode` or `PortableCode` instance. --- aiida/orm/__init__.py | 3 + aiida/orm/nodes/__init__.py | 3 + aiida/orm/nodes/data/__init__.py | 3 + aiida/orm/nodes/data/code/__init__.py | 21 ++ aiida/orm/nodes/data/code/abstract.py | 183 ++++++++++++++++++ aiida/orm/nodes/data/code/installed.py | 129 ++++++++++++ .../nodes/data/{code.py => code/legacy.py} | 2 +- aiida/orm/nodes/data/code/portable.py | 123 ++++++++++++ docs/source/conf.py | 4 +- .../source/developer_guide/core/internals.rst | 2 +- docs/source/howto/plugin_codes.rst | 2 +- docs/source/nitpick-exceptions | 2 + docs/source/topics/calculations/concepts.rst | 4 +- docs/source/topics/processes/usage.rst | 2 +- pyproject.toml | 4 +- tests/orm/data/code/test_abstract.py | 54 ++++++ tests/orm/data/code/test_installed.py | 114 +++++++++++ tests/orm/data/code/test_portable.py | 92 +++++++++ tests/orm/data/test_code.py | 2 +- 19 files changed, 739 insertions(+), 10 deletions(-) create mode 100644 aiida/orm/nodes/data/code/__init__.py create mode 100644 aiida/orm/nodes/data/code/abstract.py create mode 100644 aiida/orm/nodes/data/code/installed.py rename aiida/orm/nodes/data/{code.py => code/legacy.py} (99%) create mode 100644 aiida/orm/nodes/data/code/portable.py create mode 100644 tests/orm/data/code/test_abstract.py create mode 100644 tests/orm/data/code/test_installed.py create mode 100644 tests/orm/data/code/test_portable.py diff --git a/aiida/orm/__init__.py b/aiida/orm/__init__.py index db3d2bf518..1821d65103 100644 --- a/aiida/orm/__init__.py +++ b/aiida/orm/__init__.py @@ -28,6 +28,7 @@ __all__ = ( 'ASCENDING', + 'AbstractCode', 'AbstractNodeMeta', 'ArrayData', 'AttributeManager', @@ -60,6 +61,7 @@ 'Group', 'GroupEntityLoader', 'ImportGroup', + 'InstalledCode', 'Int', 'JsonableData', 'Kind', @@ -78,6 +80,7 @@ 'OrbitalData', 'OrderSpecifier', 'OrmEntityLoader', + 'PortableCode', 'ProcessNode', 'ProjectionData', 'QueryBuilder', diff --git a/aiida/orm/nodes/__init__.py b/aiida/orm/nodes/__init__.py index f192d2fafd..5ba5f867fc 100644 --- a/aiida/orm/nodes/__init__.py +++ b/aiida/orm/nodes/__init__.py @@ -21,6 +21,7 @@ from .repository import * __all__ = ( + 'AbstractCode', 'ArrayData', 'BandsData', 'BaseType', @@ -35,6 +36,7 @@ 'EnumData', 'Float', 'FolderData', + 'InstalledCode', 'Int', 'JsonableData', 'Kind', @@ -45,6 +47,7 @@ 'NodeRepository', 'NumericType', 'OrbitalData', + 'PortableCode', 'ProcessNode', 'ProjectionData', 'RemoteData', diff --git a/aiida/orm/nodes/data/__init__.py b/aiida/orm/nodes/data/__init__.py index df9081ed74..6d5826eccb 100644 --- a/aiida/orm/nodes/data/__init__.py +++ b/aiida/orm/nodes/data/__init__.py @@ -36,6 +36,7 @@ from .upf import * __all__ = ( + 'AbstractCode', 'ArrayData', 'BandsData', 'BaseType', @@ -47,6 +48,7 @@ 'EnumData', 'Float', 'FolderData', + 'InstalledCode', 'Int', 'JsonableData', 'Kind', @@ -54,6 +56,7 @@ 'List', 'NumericType', 'OrbitalData', + 'PortableCode', 'ProjectionData', 'RemoteData', 'RemoteStashData', diff --git a/aiida/orm/nodes/data/code/__init__.py b/aiida/orm/nodes/data/code/__init__.py new file mode 100644 index 0000000000..200466ecbc --- /dev/null +++ b/aiida/orm/nodes/data/code/__init__.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +"""Data plugins that represent an executable code.""" + +# AUTO-GENERATED + +# yapf: disable +# pylint: disable=wildcard-import + +from .abstract import * +from .installed import * +from .legacy import * +from .portable import * + +__all__ = ( + 'AbstractCode', + 'Code', + 'InstalledCode', + 'PortableCode', +) + +# yapf: enable diff --git a/aiida/orm/nodes/data/code/abstract.py b/aiida/orm/nodes/data/code/abstract.py new file mode 100644 index 0000000000..10318c3529 --- /dev/null +++ b/aiida/orm/nodes/data/code/abstract.py @@ -0,0 +1,183 @@ +# -*- coding: utf-8 -*- +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +"""Abstract data plugin representing an executable code.""" +from __future__ import annotations + +import abc + +from aiida.common.lang import type_check +from aiida.orm import Computer + +from ..data import Data + +__all__ = ('AbstractCode',) + + +class AbstractCode(Data, metaclass=abc.ABCMeta): + """Abstract data plugin representing an executable code.""" + + # Should become ``default_calc_job_plugin`` once ``Code`` is dropped in ``aiida-core==3.0`` + _KEY_ATTRIBUTE_DEFAULT_CALC_JOB_PLUGIN: str = 'input_plugin' + _KEY_ATTRIBUTE_APPEND_TEXT: str = 'append_text' + _KEY_ATTRIBUTE_PREPEND_TEXT: str = 'prepend_text' + _KEY_ATTRIBUTE_USE_DOUBLE_QUOTES: str = 'use_double_quotes' + _KEY_EXTRA_IS_HIDDEN: str = 'hidden' # Should become ``is_hidden`` once ``Code`` is dropped + + def __init__( + self, + default_calc_job_plugin: str = None, + append_text: str = '', + prepend_text: str = '', + use_double_quotes: bool = False, + is_hidden: bool = False, + **kwargs + ): + """Construct a new instance. + + :param default_calc_job_plugin: The entry point name of the default ``CalcJob`` plugin to use. + :param append_text: The text that should be appended to the run line in the job script. + :param prepend_text: The text that should be prepended to the run line in the job script. + :param use_double_quotes: Whether the command line invocation of this code should be escaped with double quotes. + :param is_hidden: Whether the code is hidden. + """ + super().__init__(**kwargs) + self.default_calc_job_plugin = default_calc_job_plugin + self.append_text = append_text + self.prepend_text = prepend_text + self.use_double_quotes = use_double_quotes + self.use_double_quotes = use_double_quotes + self.is_hidden = is_hidden + + @abc.abstractmethod + def can_run_on_computer(self, computer: Computer) -> bool: + """Return whether the code can run on a given computer. + + :param computer: The computer. + :return: ``True`` if the code can run on ``computer``, ``False`` otherwise. + """ + + @abc.abstractmethod + def get_executable(self) -> str: + """Return the executable that the submission script should execute to run the code. + + :return: The executable to be called in the submission script. + """ + + @property + @abc.abstractmethod + def full_label(self) -> str: + """Return the full label of this code. + + The full label can be just the label itself but it can be something else. However, it at the very least has to + include the label of the code. + + :return: The full label of the code. + """ + + @Data.label.setter + def label(self, value: str) -> None: + """Set the label. + + The label cannot contain any ``@`` symbols. + + :param value: The new label. + :raises ValueError: If the label contains invalid characters. + """ + type_check(value, str) + + if '@' in value: + raise ValueError('The label contains a `@` symbol, which is not allowed.') + + self.backend_entity.label = value + + @property + def default_calc_job_plugin(self) -> str | None: + """Return the optional default ``CalcJob`` plugin. + + :return: The entry point name of the default ``CalcJob`` plugin to use. + """ + return self.base.attributes.get(self._KEY_ATTRIBUTE_DEFAULT_CALC_JOB_PLUGIN, None) + + @default_calc_job_plugin.setter + def default_calc_job_plugin(self, value: str | None) -> None: + """Set the default ``CalcJob`` plugin. + + :param value: The entry point name of the default ``CalcJob`` plugin to use. + """ + type_check(value, str, allow_none=True) + self.base.attributes.set(self._KEY_ATTRIBUTE_DEFAULT_CALC_JOB_PLUGIN, value) + + @property + def append_text(self) -> str: + """Return the text that should be appended to the run line in the job script. + + :return: The text that should be appended to the run line in the job script. + """ + return self.base.attributes.get(self._KEY_ATTRIBUTE_APPEND_TEXT, '') + + @append_text.setter + def append_text(self, value: str) -> None: + """Set the text that should be appended to the run line in the job script. + + :param value: The text that should be appended to the run line in the job script. + """ + type_check(value, str, allow_none=True) + self.base.attributes.set(self._KEY_ATTRIBUTE_APPEND_TEXT, value) + + @property + def prepend_text(self) -> str: + """Return the text that should be prepended to the run line in the job script. + + :return: The text that should be prepended to the run line in the job script. + """ + return self.base.attributes.get(self._KEY_ATTRIBUTE_PREPEND_TEXT, '') + + @prepend_text.setter + def prepend_text(self, value: str) -> None: + """Set the text that should be prepended to the run line in the job script. + + :param value: The text that should be prepended to the run line in the job script. + """ + type_check(value, str, allow_none=True) + self.base.attributes.set(self._KEY_ATTRIBUTE_PREPEND_TEXT, value) + + @property + def use_double_quotes(self) -> bool: + """Return whether the command line invocation of this code should be escaped with double quotes. + + :return: ``True`` if to escape with double quotes, ``False`` otherwise. + """ + return self.base.attributes.get(self._KEY_ATTRIBUTE_USE_DOUBLE_QUOTES, False) + + @use_double_quotes.setter + def use_double_quotes(self, value: bool) -> None: + """Set whether the command line invocation of this code should be escaped with double quotes. + + :param value: ``True`` if to escape with double quotes, ``False`` otherwise. + """ + type_check(value, bool) + self.base.attributes.set(self._KEY_ATTRIBUTE_USE_DOUBLE_QUOTES, value) + + @property + def is_hidden(self) -> bool: + """Return whether the code is hidden. + + :return: ``True`` if the code is hidden, ``False`` otherwise, which is also the default. + """ + return self.base.extras.get(self._KEY_EXTRA_IS_HIDDEN, False) + + @is_hidden.setter + def is_hidden(self, value: bool) -> None: + """Define whether the code is hidden or not. + + :param value: ``True`` if the code should be hidden, ``False`` otherwise. + """ + type_check(value, bool) + self.base.extras.set(self._KEY_EXTRA_IS_HIDDEN, value) diff --git a/aiida/orm/nodes/data/code/installed.py b/aiida/orm/nodes/data/code/installed.py new file mode 100644 index 0000000000..d4b1d76365 --- /dev/null +++ b/aiida/orm/nodes/data/code/installed.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +"""Data plugin representing an executable code on a remote computer. + +This plugin should be used if an executable is pre-installed on a computer. The ``InstalledCode`` represents the code by +storing the absolute filepath of the relevant executable and the computer on which it is installed. The computer is +represented by an instance of :class:`aiida.orm.computers.Computer`. Each time a :class:`aiida.engine.CalcJob` is run +using an ``InstalledCode``, it will run its executable on the associated computer. +""" +import pathlib + +from aiida.common import exceptions +from aiida.common.lang import type_check +from aiida.common.log import override_log_level +from aiida.orm import Computer + +from .abstract import AbstractCode + +__all__ = ('InstalledCode',) + + +class InstalledCode(AbstractCode): + """Data plugin representing an executable code on a remote computer.""" + + _KEY_ATTRIBUTE_FILEPATH_EXECUTABLE: str = 'filepath_executable' + + def __init__(self, computer: Computer, filepath_executable: str, **kwargs): + """Construct a new instance. + + :param computer: The remote computer on which the executable is located. + :param filepath_executable: The absolute filepath of the executable on the remote computer. + """ + super().__init__(**kwargs) + self.computer = computer + self.filepath_executable = filepath_executable + + def _validate(self): + """Validate the instance by checking that a computer has been defined. + + :raises :class:`aiida.common.exceptions.ValidationError`: If the state of the node is invalid. + """ + super()._validate() + + if not self.computer: + raise exceptions.ValidationError('The `computer` is undefined.') + + try: + self.filepath_executable + except TypeError as exception: + raise exceptions.ValidationError('The `filepath_executable` is not set.') from exception + + def validate_filepath_executable(self): + """Validate the ``filepath_executable`` attribute. + + Checks whether the executable exists on the remote computer if a transport can be opened to it. This method + is intentionally not called in ``_validate`` as to allow the creation of ``Code`` instances whose computers can + not yet be connected to and as to not require the overhead of opening transports in storing a new code. + + :raises `~aiida.common.exceptions.ValidationError`: if no transport could be opened or if the defined executable + does not exist on the remote computer. + """ + try: + with override_log_level(): # Temporarily suppress noisy logging + with self.computer.get_transport() as transport: + file_exists = transport.isfile(self.filepath_executable) + except Exception: # pylint: disable=broad-except + raise exceptions.ValidationError( + 'Could not connect to the configured computer to determine whether the specified executable exists.' + ) + + if not file_exists: + raise exceptions.ValidationError( + f'The provided remote absolute path `{self.filepath_executable}` does not exist on the computer.' + ) + + def can_run_on_computer(self, computer: Computer) -> bool: + """Return whether the code can run on a given computer. + + :param computer: The computer. + :return: ``True`` if the provided computer is the same as the one configured for this code. + """ + type_check(computer, Computer) + return computer.pk == self.computer.pk + + def get_executable(self) -> str: + """Return the executable that the submission script should execute to run the code. + + :return: The executable to be called in the submission script. + """ + return self.filepath_executable + + @property + def full_label(self) -> str: + """Return the full label of this code. + + The full label can be just the label itself but it can be something else. However, it at the very least has to + include the label of the code. + + :return: The full label of the code. + """ + return f'{self.label}@{self.computer.label}' + + @property + def filepath_executable(self) -> pathlib.PurePath: + """Return the absolute filepath of the executable that this code represents. + + :return: The absolute filepath of the executable. + """ + return pathlib.PurePath(self.base.attributes.get(self._KEY_ATTRIBUTE_FILEPATH_EXECUTABLE)) + + @filepath_executable.setter + def filepath_executable(self, value: str) -> None: + """Set the absolute filepath of the executable that this code represents. + + :param value: The absolute filepath of the executable. + """ + type_check(value, str) + + if not pathlib.PurePath(value).is_absolute(): + raise ValueError('the `filepath_executable` should be absolute.') + + self.base.attributes.set(self._KEY_ATTRIBUTE_FILEPATH_EXECUTABLE, value) diff --git a/aiida/orm/nodes/data/code.py b/aiida/orm/nodes/data/code/legacy.py similarity index 99% rename from aiida/orm/nodes/data/code.py rename to aiida/orm/nodes/data/code/legacy.py index d5981eb734..445b0bc64a 100644 --- a/aiida/orm/nodes/data/code.py +++ b/aiida/orm/nodes/data/code/legacy.py @@ -13,7 +13,7 @@ from aiida.common import exceptions from aiida.common.log import override_log_level -from .data import Data +from ..data import Data __all__ = ('Code',) diff --git a/aiida/orm/nodes/data/code/portable.py b/aiida/orm/nodes/data/code/portable.py new file mode 100644 index 0000000000..589134c924 --- /dev/null +++ b/aiida/orm/nodes/data/code/portable.py @@ -0,0 +1,123 @@ +# -*- coding: utf-8 -*- +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +"""Data plugin representing an executable code stored in AiiDA's storage. + +This plugin should be used for executables that are not already installed on the target computer, but instead are +available on the machine where AiiDA is running. The plugin assumes that the code is self-contained by a single +directory containing all the necessary files, including a main executable. When constructing a ``PortableCode``, passing +the absolute filepath as ``filepath_files`` will make sure that all the files contained within are uploaded to AiiDA's +storage. The ``filepath_executable`` should indicate the filename of the executable within that directory. Each time a +:class:`aiida.engine.CalcJob` is run using a ``PortableCode``, the uploaded files will be automatically copied to the +working directory on the selected computer and the executable will be run there. +""" +import pathlib + +from aiida.common import exceptions +from aiida.common.lang import type_check +from aiida.orm import Computer + +from .abstract import AbstractCode + +__all__ = ('PortableCode',) + + +class PortableCode(AbstractCode): + """Data plugin representing an executable code stored in AiiDA's storage.""" + + _KEY_ATTRIBUTE_FILEPATH_EXECUTABLE: str = 'filepath_executable' + + def __init__(self, filepath_executable: str, filepath_files: pathlib.Path, **kwargs): + """Construct a new instance. + + .. note:: If the files necessary for this code are not all located in a single directory or the directory + contains files that should not be uploaded, and so the ``filepath_files`` cannot be used. One can use the + methods of the :class:`aiida.orm.nodes.repository.NodeRepository` class. This can be accessed through the + ``base.repository`` attribute of the instance after it has been constructed. For example:: + + code = PortableCode(filepath_executable='some_name.exe') + code.put_object_from_file() + code.put_object_from_filelike() + code.put_object_from_tree() + + :param filepath_executable: The relative filepath of the executable within the directory of uploaded files. + :param filepath_files: The filepath to the directory containing all the files of the code. + """ + super().__init__(**kwargs) + type_check(filepath_files, pathlib.Path) + self.filepath_executable = filepath_executable + self.base.repository.put_object_from_tree(filepath_files) + + def _validate(self): + """Validate the instance by checking that an executable is defined and it is part of the repository files. + + :raises :class:`aiida.common.exceptions.ValidationError`: If the state of the node is invalid. + """ + super()._validate() + + try: + filepath_executable = self.filepath_executable + except TypeError as exception: + raise exceptions.ValidationError('The `filepath_executable` is not set.') from exception + + objects = self.base.repository.list_object_names() + + if str(filepath_executable) not in objects: + raise exceptions.ValidationError( + f'The executable `{filepath_executable}` is not one of the uploaded files: {objects}' + ) + + def can_run_on_computer(self, computer: Computer) -> bool: + """Return whether the code can run on a given computer. + + A ``PortableCode`` should be able to be run on any computer in principle. + + :param computer: The computer. + :return: ``True`` if the provided computer is the same as the one configured for this code. + """ + return True + + def get_executable(self) -> str: + """Return the executable that the submission script should execute to run the code. + + :return: The executable to be called in the submission script. + """ + return self.filepath_executable + + @property + def full_label(self) -> str: + """Return the full label of this code. + + The full label can be just the label itself but it can be something else. However, it at the very least has to + include the label of the code. + + :return: The full label of the code. + """ + return self.label + + @property + def filepath_executable(self) -> pathlib.PurePath: + """Return the relative filepath of the executable that this code represents. + + :return: The relative filepath of the executable. + """ + return pathlib.PurePath(self.base.attributes.get(self._KEY_ATTRIBUTE_FILEPATH_EXECUTABLE)) + + @filepath_executable.setter + def filepath_executable(self, value: str) -> None: + """Set the relative filepath of the executable that this code represents. + + :param value: The relative filepath of the executable within the directory of uploaded files. + """ + type_check(value, str) + + if pathlib.PurePath(value).is_absolute(): + raise ValueError('The `filepath_executable` should not be absolute.') + + self.base.attributes.set(self._KEY_ATTRIBUTE_FILEPATH_EXECUTABLE, value) diff --git a/docs/source/conf.py b/docs/source/conf.py index 71d93945d3..a83fd17dee 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -232,7 +232,7 @@ def setup(app: Sphinx): # these are mainly required, because sphinx finds multiple references, # in `aiida.orm`` and `aiida.restapi.translator` autodoc_aliases_typing = { - 'Code': 'aiida.orm.nodes.data.code.Code', + 'Code': 'aiida.orm.nodes.data.code.legacy.Code', 'Computer': 'aiida.orm.computers.Computer', 'Data': 'aiida.orm.nodes.data.data.Data', 'Group': 'aiida.orm.groups.Group', @@ -277,7 +277,7 @@ def setup(app: Sphinx): 'aiida.orm.User': 'aiida.orm.users.User', 'aiida.orm.CalculationNode': 'aiida.orm.nodes.process.calculation.calculation.CalculationNode', 'aiida.orm.CalcJobNode': 'aiida.orm.nodes.process.calculation.calcjob.CalcJobNode', - 'aiida.orm.Code': 'aiida.orm.nodes.data.code.Code', + 'aiida.orm.Code': 'aiida.orm.nodes.data.code.legacy.Code', 'aiida.orm.Data': 'aiida.orm.nodes.data.data.Data', 'aiida.orm.Dict': 'aiida.orm.nodes.data.dict.Dict', 'aiida.orm.ProcessNode': 'aiida.orm.nodes.process.process.ProcessNode', diff --git a/docs/source/developer_guide/core/internals.rst b/docs/source/developer_guide/core/internals.rst index bb13565f09..cd0ab9e15c 100644 --- a/docs/source/developer_guide/core/internals.rst +++ b/docs/source/developer_guide/core/internals.rst @@ -6,7 +6,7 @@ Node ++++ All nodes in an AiiDA provenance graph inherit from the :py:class:`~aiida.orm.nodes.node.Node` class. -Among those are the :py:class:`~aiida.orm.nodes.data.data.Data` class, the :py:class:`~aiida.orm.nodes.process.process.ProcessNode` class representing computations that transform data, and the :py:class:`~aiida.orm.nodes.data.code.Code` class representing executables (and file collections that are used by calculations). +Among those are the :py:class:`~aiida.orm.nodes.data.data.Data` class, the :py:class:`~aiida.orm.nodes.process.process.ProcessNode` class representing computations that transform data, and the :py:class:`~aiida.orm.nodes.data.code.abstract.AbstractCode` class representing executables (and file collections that are used by calculations). Immutability concept diff --git a/docs/source/howto/plugin_codes.rst b/docs/source/howto/plugin_codes.rst index f387484af7..32c8b0c7a7 100644 --- a/docs/source/howto/plugin_codes.rst +++ b/docs/source/howto/plugin_codes.rst @@ -614,7 +614,7 @@ Continue with :ref:`how-to:plugins-develop` in order to learn how to quickly cre .. |StructureData| replace:: :py:class:`~aiida.orm.nodes.data.structure.StructureData` .. |RemoteData| replace:: :py:class:`~aiida.orm.RemoteData` .. |Dict| replace:: :py:class:`~aiida.orm.nodes.data.dict.Dict` -.. |Code| replace:: :py:class:`~aiida.orm.Code` +.. |Code| replace:: :py:class:`~aiida.orm.nodes.data.code.abstract.AbstractCode` .. |Parser| replace:: :py:class:`~aiida.parsers.parser.Parser` .. |parse| replace:: :py:class:`~aiida.parsers.parser.Parser.parse` .. |folder| replace:: :py:class:`~aiida.common.folders.Folder` diff --git a/docs/source/nitpick-exceptions b/docs/source/nitpick-exceptions index 95cc3b3a82..612300411f 100644 --- a/docs/source/nitpick-exceptions +++ b/docs/source/nitpick-exceptions @@ -24,6 +24,8 @@ py:class ReturnType py:class SelfType py:class CollectionType py:class EntityType +py:class EntityClsType +py:class ProjectType py:class BackendEntityType py:class BackendNodeType py:class TransactionType diff --git a/docs/source/topics/calculations/concepts.rst b/docs/source/topics/calculations/concepts.rst index 87d681f796..72807d47f7 100644 --- a/docs/source/topics/calculations/concepts.rst +++ b/docs/source/topics/calculations/concepts.rst @@ -127,7 +127,7 @@ the provenance generated by running the calculation job will look something like The execution of the calculation job is represented in the provenance graph by a process node, i.e. the pink square labeled `C\ :sub:`1`` in :numref:`fig_calculation_jobs_provenance_arithmetic_add`. The integer data nodes ``x`` and ``y`` that were passed as inputs are linked to the calculation job as such, as well as the third input ``code``. This input is required for *all* calculation jobs as it represents the external code that is actually executed. -These code nodes are instances of the :py:class:`~aiida.orm.nodes.data.code.Code` class, which is a sub-class of :py:class:`~aiida.orm.nodes.data.data.Data`. +These code nodes are instances of the :py:class:`~aiida.orm.nodes.data.code.abstract.AbstractCode` class, which is a sub-class of :py:class:`~aiida.orm.nodes.data.data.Data`. This means that code instances are a sort of data node. Its function is to record the path to the executable and some other code related attributes defined during the code setup. @@ -150,7 +150,7 @@ When a calculation job is launched, the engine will take it roughly through the * **Retrieve**: once the job has finished, the engine will retrieve the output files, specified by the calculation plugin and store them in a node attached as an output node to the calculation All of these tasks require the engine to interact with the computer, or machine, that will actually run the external code. -Since the :py:class:`~aiida.orm.nodes.data.code.Code` that is used as an input for the calculation job, which is configured for a specific :py:class:`~aiida.orm.computers.Computer`, the engine knows exactly how to execute all these tasks. +Since the :py:class:`~aiida.orm.nodes.data.code.abstract.AbstractCode` that is used as an input for the calculation job, which is configured for a specific :py:class:`~aiida.orm.computers.Computer`, the engine knows exactly how to execute all these tasks. The ``CalcJob`` implementation itself then is completely independent of the machine the code will be run on. To run the calculation job on a different machine, all you have to do is change the ``code`` input to one that is configured for that machine. If the machine is *not* the localhost, the engine will need a way to connect to the remote machine in order to perform each of the four tasks listed above. diff --git a/docs/source/topics/processes/usage.rst b/docs/source/topics/processes/usage.rst index c8c1861be6..3494c13f34 100644 --- a/docs/source/topics/processes/usage.rst +++ b/docs/source/topics/processes/usage.rst @@ -410,7 +410,7 @@ In an interactive shell, you can get this information to display as follows:: "name": "code", "required": "True" "non_db": "False" - "valid_type": "" + "valid_type": "" "help": "The Code to use for this job.", In the ``Docstring`` you will see a ``help`` string that contains more detailed information about the input port. diff --git a/pyproject.toml b/pyproject.toml index 33f9731913..b9ec66c940 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -167,7 +167,9 @@ runaiida = "aiida.cmdline.commands.cmd_run:run" "core.base" = "aiida.orm.nodes.data:BaseType" "core.bool" = "aiida.orm.nodes.data.bool:Bool" "core.cif" = "aiida.orm.nodes.data.cif:CifData" -"core.code" = "aiida.orm.nodes.data.code:Code" +"core.code" = "aiida.orm.nodes.data.code.legacy:Code" +"core.code.portable" = "aiida.orm.nodes.data.code.portable:PortableCode" +"core.code.installed" = "aiida.orm.nodes.data.code.installed:InstalledCode" "core.dict" = "aiida.orm.nodes.data.dict:Dict" "core.enum" = "aiida.orm.nodes.data.enum:EnumData" "core.float" = "aiida.orm.nodes.data.float:Float" diff --git a/tests/orm/data/code/test_abstract.py b/tests/orm/data/code/test_abstract.py new file mode 100644 index 0000000000..c4976d9067 --- /dev/null +++ b/tests/orm/data/code/test_abstract.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +# pylint: disable=redefined-outer-name +"""Tests for the :class:`aiida.orm.nodes.data.code.abstract.AbstractCode` class.""" +import pytest + +from aiida.orm.nodes.data.code.abstract import AbstractCode + + +class MockCode(AbstractCode): + """Implementation of :class:`aiida.orm.nodes.data.code.abstract.AbstractCode`.""" + + def can_run_on_computer(self, computer) -> bool: + """Return whether the code can run on a given computer.""" + return True + + def get_executable(self) -> str: + """Return the executable that the submission script should execute to run the code.""" + return '' + + @property + def full_label(self) -> str: + """Return the full label of this code.""" + return '' + + +def test_set_label(): + """Test the :meth:`aiida.orm.nodes.data.code.abstract.AbstractCode.label` property setter.""" + label = 'some-label' + code = MockCode(label=label) + assert code.label == label + + code.label = 'alternate-label' + assert code.label == 'alternate-label' + + with pytest.raises(ValueError, match=''): + code.label = 'illegal@label' + + +def test_constructor_defaults(): + """Test the defaults of the constructor.""" + code = MockCode() + assert code.default_calc_job_plugin is None + assert code.append_text == '' + assert code.prepend_text == '' + assert code.use_double_quotes is False + assert code.is_hidden is False diff --git a/tests/orm/data/code/test_installed.py b/tests/orm/data/code/test_installed.py new file mode 100644 index 0000000000..124579325f --- /dev/null +++ b/tests/orm/data/code/test_installed.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +# pylint: disable=redefined-outer-name +"""Tests for the :class:`aiida.orm.nodes.data.code.installed.InstalledCode` class.""" +import pathlib +import uuid + +import pytest + +from aiida.common.exceptions import ModificationNotAllowed, ValidationError +from aiida.orm import Computer +from aiida.orm.nodes.data.code.installed import InstalledCode + + +def test_constructor_raises(aiida_localhost): + """Test the constructor when it is supposed to raise.""" + with pytest.raises(TypeError, match=r'missing .* required positional arguments'): + InstalledCode() # pylint: disable=no-value-for-parameter + + with pytest.raises(TypeError, match=r'Got object of type .*'): + InstalledCode(computer=aiida_localhost, filepath_executable=pathlib.Path('/usr/bin/bash')) + + with pytest.raises(TypeError, match=r'Got object of type .*'): + InstalledCode(computer='computer', filepath_executable='/usr/bin/bash') + + +def test_constructor(aiida_localhost): + """Test the constructor.""" + filepath_executable = '/usr/bin/bash' + code = InstalledCode(computer=aiida_localhost, filepath_executable=filepath_executable) + assert code.computer.pk == aiida_localhost.pk + assert code.filepath_executable == pathlib.PurePath(filepath_executable) + + +def test_validate(aiida_localhost): + """Test the validator is called before storing.""" + filepath_executable = '/usr/bin/bash' + code = InstalledCode(computer=aiida_localhost, filepath_executable=filepath_executable) + + code.computer = None + assert code.computer is None + + with pytest.raises(ValidationError, match='The `computer` is undefined.'): + code.store() + + code.computer = aiida_localhost + code.base.attributes.set(code._KEY_ATTRIBUTE_FILEPATH_EXECUTABLE, None) # pylint: disable=protected-access + + with pytest.raises(ValidationError, match='The `filepath_executable` is not set.'): + code.store() + + code.filepath_executable = filepath_executable + code.store() + assert code.is_stored + + +def test_can_run_on_computer(aiida_localhost): + """Test the :meth:`aiida.orm.nodes.data.code.installed.InstalledCode.can_run_on_computer` method.""" + code = InstalledCode(computer=aiida_localhost, filepath_executable='/usr/bin/bash') + computer = Computer() + + assert code.can_run_on_computer(aiida_localhost) + assert not code.can_run_on_computer(computer) + + +def test_filepath_executable(aiida_localhost): + """Test the :meth:`aiida.orm.nodes.data.code.installed.InstalledCode.filepath_executable` property.""" + filepath_executable = '/usr/bin/bash' + code = InstalledCode(computer=aiida_localhost, filepath_executable=filepath_executable) + assert code.filepath_executable == pathlib.PurePath(filepath_executable) + + filepath_executable = '/usr/bin/cat' + code.filepath_executable = filepath_executable + assert code.filepath_executable == pathlib.PurePath(filepath_executable) + + with pytest.raises(TypeError, match=r'Got object of type .*'): + code.filepath_executable = pathlib.Path(filepath_executable) + + code.store() + + with pytest.raises(ModificationNotAllowed): + code.filepath_executable = filepath_executable + + +def test_validate_filepath_executable(): + """Test the :meth:`aiida.orm.nodes.data.code.installed.InstalledCode.validate_filepath_executable` method.""" + filepath_executable = '/usr/bin/not-existing' + computer = Computer(label=str(uuid.uuid4()), transport_type='core.local') + code = InstalledCode(computer=computer, filepath_executable=filepath_executable) + + with pytest.raises(ValidationError, match=r'Could not connect to the configured computer.*'): + code.validate_filepath_executable() + + computer.configure() + + with pytest.raises(ValidationError, match=r'The provided remote absolute path .* does not exist on the computer\.'): + code.validate_filepath_executable() + + code.filepath_executable = '/usr/bin/bash' + code.validate_filepath_executable() + + +def test_full_label(aiida_localhost): + """Test the :meth:`aiida.orm.nodes.data.code.portable.PortableCode.full_label` property.""" + label = 'some-label' + code = InstalledCode(label=label, computer=aiida_localhost, filepath_executable='/usr/bin/bash') + assert code.full_label == f'{label}@{aiida_localhost.label}' diff --git a/tests/orm/data/code/test_portable.py b/tests/orm/data/code/test_portable.py new file mode 100644 index 0000000000..8c49d06aaa --- /dev/null +++ b/tests/orm/data/code/test_portable.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +# pylint: disable=redefined-outer-name +"""Tests for the :class:`aiida.orm.nodes.data.code.portable.PortableCode` class.""" +import io +import pathlib + +import pytest + +from aiida.common.exceptions import ModificationNotAllowed, ValidationError +from aiida.orm.nodes.data.code.portable import PortableCode + + +def test_constructor_raises(tmp_path): + """Test the constructor when it is supposed to raise.""" + with pytest.raises(TypeError, match=r'missing .* required positional argument'): + PortableCode() # pylint: disable=no-value-for-parameter + + with pytest.raises(TypeError, match=r'Got object of type .*'): + PortableCode(filepath_executable=pathlib.Path('/usr/bin/bash'), filepath_files=tmp_path) + + with pytest.raises(TypeError, match=r'Got object of type .*'): + PortableCode(filepath_executable='bash', filepath_files='string') + + +def test_constructor(tmp_path): + """Test the constructor.""" + (tmp_path / 'bash').touch() + (tmp_path / 'alternate').touch() + filepath_executable = 'bash' + code = PortableCode(filepath_executable=filepath_executable, filepath_files=tmp_path) + assert code.filepath_executable == pathlib.PurePath(filepath_executable) + assert sorted(code.base.repository.list_object_names()) == ['alternate', 'bash'] + + +def test_validate(tmp_path): + """Test the validator is called before storing.""" + filepath_executable = 'bash' + code = PortableCode(filepath_executable=filepath_executable, filepath_files=tmp_path) + + code.base.attributes.set(code._KEY_ATTRIBUTE_FILEPATH_EXECUTABLE, None) # pylint: disable=protected-access + + with pytest.raises(ValidationError, match='The `filepath_executable` is not set.'): + code.store() + + code.filepath_executable = filepath_executable + + with pytest.raises(ValidationError, match=r'The executable .* is not one of the uploaded files'): + code.store() + + code.base.repository.put_object_from_filelike(io.BytesIO(b''), filepath_executable) + code.store() + assert code.is_stored + + +def test_can_run_on_computer(aiida_localhost, tmp_path): + """Test the :meth:`aiida.orm.nodes.data.code.portable.PortableCode.can_run_on_computer` method.""" + code = PortableCode(filepath_executable='./bash', filepath_files=tmp_path) + assert code.can_run_on_computer(aiida_localhost) + + +def test_filepath_executable(tmp_path): + """Test the :meth:`aiida.orm.nodes.data.code.portable.PortableCode.filepath_executable` property.""" + filepath_executable = 'bash' + code = PortableCode(filepath_executable=filepath_executable, filepath_files=tmp_path) + + with pytest.raises(ValueError, match=r'The `filepath_executable` should not be absolute.'): + code.filepath_executable = '/usr/bin/cat' + + with pytest.raises(TypeError, match=r'Got object of type .*'): + code.filepath_executable = pathlib.Path(filepath_executable) + + code.filepath_executable = filepath_executable + code.base.repository.put_object_from_filelike(io.BytesIO(b''), filepath_executable) + code.store() + + with pytest.raises(ModificationNotAllowed): + code.filepath_executable = filepath_executable + + +def test_full_label(tmp_path): + """Test the :meth:`aiida.orm.nodes.data.code.portable.PortableCode.full_label` property.""" + label = 'some-label' + code = PortableCode(label=label, filepath_executable='bash', filepath_files=tmp_path) + assert code.full_label == label diff --git a/tests/orm/data/test_code.py b/tests/orm/data/test_code.py index 6b5864229e..2b06c58698 100644 --- a/tests/orm/data/test_code.py +++ b/tests/orm/data/test_code.py @@ -8,7 +8,7 @@ # For further information please visit http://www.aiida.net # ########################################################################### # pylint: disable=redefined-outer-name -"""Tests for :class:`aiida.orm.nodes.data.code.Code` class.""" +"""Tests for :class:`aiida.orm.nodes.data.code.legacy.Code` class.""" import pytest from aiida.common.exceptions import ValidationError From aa1e324803c4a14f18b0347fda91b19c1a2682d4 Mon Sep 17 00:00:00 2001 From: Sebastiaan Huber Date: Mon, 2 May 2022 23:15:33 +0200 Subject: [PATCH 2/7] Add CLI options for all code implementations The `AbstractCode` class implements the `get_cli_options` method which will return an `OrderedDict` of the dictionary returned by the `_get_cli_options`. This dict contains a spec definition of the CLI options a command should have to construct an instance of the relevant data plugin class. The `InstalleCode` and `PortableCode` add their specific options to this by overriding the `_get_cli_options` method. With this approach, the `verdi` CLI could dynamically add commands that allow to create their instances by discovering them through their entry points and building up the command options by the spec returned by `get_cli_options`. Each entry in the dictionary returned by `get_cli_options` will essentially be passed to the `click.option` decorator, so its keys should match the spec of that method. --- aiida/orm/nodes/data/code/abstract.py | 105 +++++++++++++++++++++++++ aiida/orm/nodes/data/code/installed.py | 47 +++++++++++ aiida/orm/nodes/data/code/portable.py | 23 ++++++ 3 files changed, 175 insertions(+) diff --git a/aiida/orm/nodes/data/code/abstract.py b/aiida/orm/nodes/data/code/abstract.py index 10318c3529..c267e94ac9 100644 --- a/aiida/orm/nodes/data/code/abstract.py +++ b/aiida/orm/nodes/data/code/abstract.py @@ -11,9 +11,15 @@ from __future__ import annotations import abc +import collections +import click + +from aiida.cmdline.params.options.interactive import TemplateInteractiveOption +from aiida.common import exceptions from aiida.common.lang import type_check from aiida.orm import Computer +from aiida.plugins import CalculationFactory from ..data import Data @@ -181,3 +187,102 @@ def is_hidden(self, value: bool) -> None: """ type_check(value, bool) self.base.extras.set(self._KEY_EXTRA_IS_HIDDEN, value) + + def get_builder(self): + """Create and return a new ``ProcessBuilder`` for the ``CalcJob`` class of the plugin configured for this code. + + The configured calculation plugin class is defined by the ``default_calc_job_plugin`` property. + + .. note:: it also sets the ``builder.code`` value. + + :return: a ``ProcessBuilder`` instance with the ``code`` input already populated with ourselves + :raise aiida.common.EntryPointError: if the specified plugin does not exist. + :raise ValueError: if no default plugin was specified. + """ + entry_point = self.default_calc_job_plugin + + if entry_point is None: + raise ValueError('No default calculation input plugin specified for this code') + + try: + process_class = CalculationFactory(entry_point) + except exceptions.EntryPointError: + raise exceptions.EntryPointError(f'The calculation entry point `{entry_point}` could not be loaded') + + builder = process_class.get_builder() + builder.code = self + + return builder + + @staticmethod + def cli_validate_label_uniqueness(_, __, value): + """Validate the uniqueness of the label of the code.""" + from aiida.orm import load_code + + try: + load_code(value) + except exceptions.NotExistent: + pass + except exceptions.MultipleObjectsError: + raise click.BadParameter(f'Multiple codes with the label `{value}` already exist.') + else: + raise click.BadParameter(f'A code with the label `{value}` already exists.') + + return value + + @classmethod + def get_cli_options(cls) -> collections.OrderedDict: + """Return the CLI options that would allow to create an instance of this class.""" + return collections.OrderedDict(cls._get_cli_options()) + + @classmethod + def _get_cli_options(cls) -> dict: + """Return the CLI options that would allow to create an instance of this class.""" + return { + 'label': { + 'required': True, + 'type': click.STRING, + 'prompt': 'Label', + 'help': 'A unique label to identify the code by.', + 'callback': cls.cli_validate_label_uniqueness, + }, + 'description': { + 'type': click.STRING, + 'prompt': 'Description', + 'help': 'Human-readable description of this code ideally including version and compilation environment.' + }, + 'default_calc_job_plugin': { + 'type': click.STRING, + 'prompt': 'Default `CalcJob` plugin', + 'help': 'Entry point name of the default plugin (as listed in `verdi plugin list aiida.calculations`).' + }, + 'prepend_text': { + 'cls': TemplateInteractiveOption, + 'type': click.STRING, + 'default': '', + 'prompt': 'Prepend script', + 'help': 'Bash commands that should be prepended to the run line in all submit scripts for this code.', + 'extension': '.bash', + 'header': 'PREPEND_TEXT: if there is any bash commands that should be prepended to the executable call ' + 'in all submit scripts for this code, type that between the equal signs below and save the file.', + 'footer': 'All lines that start with `#=`: will be ignored.' + }, + 'append_text': { + 'cls': TemplateInteractiveOption, + 'type': click.STRING, + 'default': '', + 'prompt': 'Append script', + 'help': 'Bash commands that should be appended to the run line in all submit scripts for this code.', + 'extension': '.bash', + 'header': 'APPEND_TEXT: if there is any bash commands that should be appended to the executable call ' + 'in all submit scripts for this code, type that between the equal signs below and save the file.', + 'footer': 'All lines that start with `#=`: will be ignored.' + }, + 'use_double_quotes': { + 'is_flag': True, + 'default': False, + 'help': 'Whether the executable and arguments of the code in the submission script should be escaped ' + 'with single or double quotes.', + 'prompt': 'Escape using double quotes', + }, + } diff --git a/aiida/orm/nodes/data/code/installed.py b/aiida/orm/nodes/data/code/installed.py index d4b1d76365..ea4f26f235 100644 --- a/aiida/orm/nodes/data/code/installed.py +++ b/aiida/orm/nodes/data/code/installed.py @@ -16,6 +16,9 @@ """ import pathlib +import click + +from aiida.cmdline.params.types import ComputerParamType from aiida.common import exceptions from aiida.common.lang import type_check from aiida.common.log import override_log_level @@ -127,3 +130,47 @@ def filepath_executable(self, value: str) -> None: raise ValueError('the `filepath_executable` should be absolute.') self.base.attributes.set(self._KEY_ATTRIBUTE_FILEPATH_EXECUTABLE, value) + + @staticmethod + def cli_validate_label_uniqueness(ctx, _, value): + """Validate the uniqueness of the label of the code.""" + from aiida.orm import load_code + + computer = ctx.params.get('computer', None) + + if computer is None: + return value + + full_label = f'{value}@{computer.label}' + + try: + load_code(full_label) + except exceptions.NotExistent: + pass + except exceptions.MultipleObjectsError: + raise click.BadParameter(f'Multiple codes with the label `{full_label}` already exist.') + else: + raise click.BadParameter(f'A code with the label `{full_label}` already exists.') + + return value + + @classmethod + def _get_cli_options(cls) -> dict: + """Return the CLI options that would allow to create an instance of this class.""" + options = { + 'computer': { + 'required': True, + 'prompt': 'Computer', + 'help': 'The remote computer on which the executable resides.', + 'type': ComputerParamType(), + }, + 'filepath_executable': { + 'required': True, + 'type': click.Path(exists=False), + 'prompt': 'Absolute filepath executable', + 'help': 'Absolute filepath of the executable on the remote computer.', + } + } + options.update(**super()._get_cli_options()) + + return options diff --git a/aiida/orm/nodes/data/code/portable.py b/aiida/orm/nodes/data/code/portable.py index 589134c924..613e76bc4b 100644 --- a/aiida/orm/nodes/data/code/portable.py +++ b/aiida/orm/nodes/data/code/portable.py @@ -19,6 +19,8 @@ """ import pathlib +import click + from aiida.common import exceptions from aiida.common.lang import type_check from aiida.orm import Computer @@ -121,3 +123,24 @@ def filepath_executable(self, value: str) -> None: raise ValueError('The `filepath_executable` should not be absolute.') self.base.attributes.set(self._KEY_ATTRIBUTE_FILEPATH_EXECUTABLE, value) + + @classmethod + def _get_cli_options(cls) -> dict: + """Return the CLI options that would allow to create an instance of this class.""" + options = { + 'filepath_executable': { + 'required': True, + 'type': click.STRING, + 'prompt': 'Relative filepath executable', + 'help': 'Relative filepath of executable with directory of code files.', + }, + 'filepath_files': { + 'required': True, + 'type': click.Path(exists=True, file_okay=False, dir_okay=True, path_type=pathlib.Path), + 'prompt': 'Code directory', + 'help': 'Filepath to directory containing code files.', + } + } + options.update(**super()._get_cli_options()) + + return options From 1ec894ac971d04812b2790371ac824bc50241ddf Mon Sep 17 00:00:00 2001 From: Sebastiaan Huber Date: Mon, 2 May 2022 20:37:42 +0200 Subject: [PATCH 3/7] CLI: Add the `DynamicEntryPointCommandGroup` command group This custom implementation of `click.Group` will allow a CLI command to dynamically generate its subcommands. The subcommands will be based on the installed entry points for a specific entry point group. Optionally, the entry points can be filtered using a regex pattern. This approach was already in use to dynamically provide the commands to configure computers with the various transport type plugins. The functionality is generalized and added as a single class to the `aiida.cmdline.groups.dynamic` module. The code base already had one other implementation of `Group`, namely the `VerdiCommandGroup`. For consistency, this is now also moved from `aiida.cmdline.commands.cmd_verdi` to `aiida.cmdline.groups.verdi`. --- aiida/cmdline/__init__.py | 3 + aiida/cmdline/commands/cmd_verdi.py | 91 +--------------------- aiida/cmdline/groups/__init__.py | 17 ++++ aiida/cmdline/groups/dynamic.py | 116 ++++++++++++++++++++++++++++ aiida/cmdline/groups/verdi.py | 100 ++++++++++++++++++++++++ aiida/plugins/entry_point.py | 30 +++---- docs/source/topics/plugins.rst | 2 +- 7 files changed, 254 insertions(+), 105 deletions(-) create mode 100644 aiida/cmdline/groups/__init__.py create mode 100644 aiida/cmdline/groups/dynamic.py create mode 100644 aiida/cmdline/groups/verdi.py diff --git a/aiida/cmdline/__init__.py b/aiida/cmdline/__init__.py index 1eed47d50e..f34e1f5f52 100644 --- a/aiida/cmdline/__init__.py +++ b/aiida/cmdline/__init__.py @@ -14,6 +14,7 @@ # yapf: disable # pylint: disable=wildcard-import +from .groups import * from .params import * from .utils import * @@ -24,6 +25,7 @@ 'ComputerParamType', 'ConfigOptionParamType', 'DataParamType', + 'DynamicEntryPointCommandGroup', 'EmailType', 'EntryPointType', 'FileOrUrl', @@ -42,6 +44,7 @@ 'ProfileParamType', 'ShebangParamType', 'UserParamType', + 'VerdiCommandGroup', 'WorkflowParamType', 'dbenv', 'echo_critical', diff --git a/aiida/cmdline/commands/cmd_verdi.py b/aiida/cmdline/commands/cmd_verdi.py index 959c9f4ec9..bcbdbb11ee 100644 --- a/aiida/cmdline/commands/cmd_verdi.py +++ b/aiida/cmdline/commands/cmd_verdi.py @@ -8,99 +8,12 @@ # For further information please visit http://www.aiida.net # ########################################################################### """The main `verdi` click group.""" -import difflib - import click from aiida import __version__ -from aiida.cmdline.params import options, types - -GIU = ( - 'ABzY8%U8Kw0{@klyK?I~3`Ki?#qHQ&IIM|J;6yB`9_+{&w)p(JK}vokj-11jhve8xcx?dZ>+9nwrEF!x*S>9A+EWYrR?6GA-u?jFa+et65GF@1+D{%' - '8{C~xjt%>uVM4RTSS?j2M)XH%T#>M{K$lE2XGD`RS0T67213wbAs!SZmn+;(-m!>f(T@e%@oxd`yRBp9nu+9N`4xv8AS@O$CaQ;7FXzM=ug^$?3ta2551EDL`wK4|Cm' - '%RnJdS#0UFwVweDkcfdNjtUv1N^iSQui#TL(q!FmIeKb!yW4|L`@!@-4x6' - 'B6I^ptRdH+4o0ODM;1_f^}4@LMe@#_YHz0wQdq@d)@n)uYNtAb2OLo&fpBkct5{~3kbRag^_5QG%qrTksHMXAYAQoz1#2wtHCy0}h?CJtzv&@Q?^9r' - 'd&02;isB7NJMMr7F@>$!ELj(sbwzIR4)rnch=oVZrG;8)%R6}FUk*fv2O&!#ZA)$HloK9!es&4Eb+h=OIyWFha(8PPy9u?NqfkuPYg;GO1RVzBLX)7' - 'ORMM>1hEM`-96mGjJ+A!e-_}4X{M|4CkKE~uF4j+LW#6IsFa*_da_mLqzr)E<`%ikthkMO2>65cNMtpDE*VejqZV^MyewPJJAS*VM6jY;QY' - '#g7gOKgPbFg{@;YDL6Gbxxr|2T&BQunB?PBetq?X>jW1hFF7&>EaYkKYqIa_ld(Z@AJT' - '+lJ(Pd;+?<&&M>A0agti19^z3n4Z6_WG}c~_+XHyJI_iau7+V$#YA$pJ~H)yHEVy1D?5^Sw`tb@{nnNNo=eSMZLf0>m^A@7f{y$nb_HJWgLRtZ?x2?*>SwM?JoQ>p|-1ZRU0#+{^UhK22+~o' - 'R9k7rh(GH9y|jm){jY9_xAI4N_EfU#4' - 'taTUXFY4a4l$v=N-+f+w&wuH;Z(6p6#=n8XwlZ;*L&-rcL~T_vEm@#-Xi8&g06!MO+R( list[str]: + """Return the sorted list of subcommands for this group. + + :param ctx: The :class:`click.Context`. + """ + commands = super().list_commands(ctx) + commands.extend([ + entry_point for entry_point in get_entry_point_names(self.entry_point_group) + if re.match(self.entry_point_name_filter, entry_point) + ]) + return sorted(commands) + + def get_command(self, ctx, cmd_name): + """Return the command with the given name. + + :param ctx: The :class:`click.Context`. + :param cmd_name: The name of the command. + :returns: The :class:`click.Command`. + """ + try: + command = self.create_command(cmd_name) + except exceptions.EntryPointError: + command = super().get_command(ctx, cmd_name) + return command + + def create_command(self, entry_point): + """Create a subcommand to create an instance of a particular code subclass""" + cls = self.factory(entry_point) + + def command(non_interactive, **kwargs): # pylint: disable=unused-argument + """Create a new instance.""" + try: + instance = cls(**kwargs) + except Exception as exception: # pylint: disable=broad-except + echo.echo_critical(f'Failed to instantiate `{cls}`: {exception}') + + try: + instance.store() + except Exception as exception: # pylint: disable=broad-except + echo.echo_critical(f'Failed to store instance of `{cls}`: {exception}') + + echo.echo_success(f'Created {cls.__name__}<{instance.pk}>') + + command.__doc__ = cls.__doc__ + + return click.command(entry_point)(self.create_options(entry_point)(command)) + + def create_options(self, entry_point): + """Create the option decorators for the command function for the given entry point. + + :param entry_point: The entry point. + """ + + def apply_options(func): + """Decorate the command function with the appropriate options for the given entry point.""" + func = options.NON_INTERACTIVE()(func) + func = options.CONFIG_FILE()(func) + + options_list = self.list_options(entry_point) + options_list.reverse() + + for option in options_list: + func = option(func) + + return func + + return apply_options + + def list_options(self, entry_point): + """Return the list of options that should be applied to the command for the given entry point. + + :param entry_point: The entry point. + """ + return [self.create_option(*item) for item in self.factory(entry_point).get_cli_options().items()] + + @staticmethod + def create_option(name, spec): + """Create a click option from a name and a specification.""" + spec = copy.deepcopy(spec) + + is_flag = spec.pop('is_flag', False) + name_dashed = name.replace('_', '-') + option_name = f'--{name_dashed}/--no-{name_dashed}' if is_flag else f'--{name_dashed}' + + kwargs = {'cls': spec.pop('cls', InteractiveOption), 'show_default': True, 'is_flag': is_flag, **spec} + + return click.option(option_name, **kwargs) diff --git a/aiida/cmdline/groups/verdi.py b/aiida/cmdline/groups/verdi.py new file mode 100644 index 0000000000..2937389cd0 --- /dev/null +++ b/aiida/cmdline/groups/verdi.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +"""Subclass of :class:`click.Group` for the ``verdi`` CLI.""" +import base64 +import difflib +import gzip + +import click + +from ..params import options + +__all__ = ('VerdiCommandGroup',) + +GIU = ( + 'ABzY8%U8Kw0{@klyK?I~3`Ki?#qHQ&IIM|J;6yB`9_+{&w)p(JK}vokj-11jhve8xcx?dZ>+9nwrEF!x*S>9A+EWYrR?6GA-u?jFa+et65GF@1+D{%' + '8{C~xjt%>uVM4RTSS?j2M)XH%T#>M{K$lE2XGD`RS0T67213wbAs!SZmn+;(-m!>f(T@e%@oxd`yRBp9nu+9N`4xv8AS@O$CaQ;7FXzM=ug^$?3ta2551EDL`wK4|Cm' + '%RnJdS#0UFwVweDkcfdNjtUv1N^iSQui#TL(q!FmIeKb!yW4|L`@!@-4x6' + 'B6I^ptRdH+4o0ODM;1_f^}4@LMe@#_YHz0wQdq@d)@n)uYNtAb2OLo&fpBkct5{~3kbRag^_5QG%qrTksHMXAYAQoz1#2wtHCy0}h?CJtzv&@Q?^9r' + 'd&02;isB7NJMMr7F@>$!ELj(sbwzIR4)rnch=oVZrG;8)%R6}FUk*fv2O&!#ZA)$HloK9!es&4Eb+h=OIyWFha(8PPy9u?NqfkuPYg;GO1RVzBLX)7' + 'ORMM>1hEM`-96mGjJ+A!e-_}4X{M|4CkKE~uF4j+LW#6IsFa*_da_mLqzr)E<`%ikthkMO2>65cNMtpDE*VejqZV^MyewPJJAS*VM6jY;QY' + '#g7gOKgPbFg{@;YDL6Gbxxr|2T&BQunB?PBetq?X>jW1hFF7&>EaYkKYqIa_ld(Z@AJT' + '+lJ(Pd;+?<&&M>A0agti19^z3n4Z6_WG}c~_+XHyJI_iau7+V$#YA$pJ~H)yHEVy1D?5^Sw`tb@{nnNNo=eSMZLf0>m^A@7f{y$nb_HJWgLRtZ?x2?*>SwM?JoQ>p|-1ZRU0#+{^UhK22+~o' + 'R9k7rh(GH9y|jm){jY9_xAI4N_EfU#4' + 'taTUXFY4a4l$v=N-+f+w&wuH;Z(6p6#=n8XwlZ;*L&-rcL~T_vEm@#-Xi8&g06!MO+R( EntryPoint: """Return an entry point, given its group and spec (as formatted in the setup)""" @@ -105,21 +119,7 @@ def validate_registered_entry_points() -> None: # pylint: disable=invalid-name * The resource's type is incompatible with the entry point group that it is defined in. """ - from . import factories - - factory_mapping = { - 'aiida.calculations': factories.CalculationFactory, - 'aiida.data': factories.DataFactory, - 'aiida.groups': factories.GroupFactory, - 'aiida.parsers': factories.ParserFactory, - 'aiida.schedulers': factories.SchedulerFactory, - 'aiida.transports': factories.TransportFactory, - 'aiida.tools.dbimporters': factories.DbImporterFactory, - 'aiida.tools.data.orbital': factories.OrbitalFactory, - 'aiida.workflows': factories.WorkflowFactory, - } - - for entry_point_group, factory in factory_mapping.items(): + for entry_point_group, factory in ENTRY_POINT_GROUP_FACTORY_MAPPING.items(): entry_points = get_entry_points(entry_point_group) for entry_point in entry_points: factory(entry_point.name) diff --git a/docs/source/topics/plugins.rst b/docs/source/topics/plugins.rst index 5633b85539..cbb1b1ad41 100644 --- a/docs/source/topics/plugins.rst +++ b/docs/source/topics/plugins.rst @@ -212,7 +212,7 @@ Usage:: ----------------- ``verdi`` uses the `click_` framework, which makes it possible to add new subcommands to existing verdi commands, such as ``verdi data mydata``. -AiiDA expects each entry point to be either a ``click.Command`` or ``click.CommandGroup``. At present extra commands can be injected at the following levels: +AiiDA expects each entry point to be either a ``click.Command`` or ``click.Group``. At present extra commands can be injected at the following levels: * As a :ref:`direct subcommand of verdi data` * As a :ref:`subcommand of verdi data structure import` From 4f79e003cb2972f0972a0e96b281e1155a9c8c09 Mon Sep 17 00:00:00 2001 From: Sebastiaan Huber Date: Mon, 2 May 2022 23:34:34 +0200 Subject: [PATCH 4/7] CLI: Add the `verdi code create` command group This is the successor of `verdi code setup`. The difference being that where `verdi code setup` hardcoded the options and was designed to setup both remote and local codes, `verdi code create` is a command group that dynamically loads subcommands for each installed entry point that is a subclass of the `AbstractCode` data plugin. The advantage of the new approach is that it is much clearer for both users as well as developers. The code and user interface for the code setup command was getting quite complicated because a single command was being used for two different purposes. By decoupling them, the interface is much more intuitive and the code is much more easy to maintain. --- aiida/cmdline/commands/cmd_code.py | 6 ++++++ docs/source/reference/command_line.rst | 1 + tests/cmdline/commands/test_code.py | 28 ++++++++++++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/aiida/cmdline/commands/cmd_code.py b/aiida/cmdline/commands/cmd_code.py index 6b5f07c854..e83edd0a5c 100644 --- a/aiida/cmdline/commands/cmd_code.py +++ b/aiida/cmdline/commands/cmd_code.py @@ -14,6 +14,7 @@ import tabulate from aiida.cmdline.commands.cmd_verdi import verdi +from aiida.cmdline.groups.dynamic import DynamicEntryPointCommandGroup from aiida.cmdline.params import arguments, options from aiida.cmdline.params.options.commands import code as options_code from aiida.cmdline.utils import echo @@ -26,6 +27,11 @@ def verdi_code(): """Setup and manage codes.""" +@verdi_code.group('create', cls=DynamicEntryPointCommandGroup, entry_point_name_filter=r'core\.code\..*') +def code_create(): + """Create a new code.""" + + def get_default(key, ctx): """ Get the default argument using a user instance property diff --git a/docs/source/reference/command_line.rst b/docs/source/reference/command_line.rst index 4a1adab63e..4d2991ea18 100644 --- a/docs/source/reference/command_line.rst +++ b/docs/source/reference/command_line.rst @@ -71,6 +71,7 @@ Below is a list with all available subcommands. --help Show this message and exit. Commands: + create Create a new code. delete Delete a code. duplicate Duplicate a code allowing to change some parameters. hide Hide one or more codes from `verdi code list`. diff --git a/tests/cmdline/commands/test_code.py b/tests/cmdline/commands/test_code.py index 6ab882cd30..38898a93ba 100644 --- a/tests/cmdline/commands/test_code.py +++ b/tests/cmdline/commands/test_code.py @@ -12,6 +12,7 @@ import os import tempfile import textwrap +import uuid import click import pytest @@ -20,6 +21,7 @@ from aiida.cmdline.params.options.commands.code import validate_label_uniqueness from aiida.common.exceptions import MultipleObjectsError, NotExistent from aiida.orm import Code, Computer, load_code +from aiida.plugins import DataFactory @pytest.fixture @@ -406,3 +408,29 @@ def test_code_test(run_cli_command): code = Code(remote_computer_exec=[computer, '/bin/bash']).store() result = run_cli_command(cmd_code.code_test, [str(code.pk)]) assert 'all tests succeeded.' in result.output + + +@pytest.fixture +def command_options(request, aiida_localhost, tmp_path): + """Return tuple of list of options and entry point.""" + options = [request.param, '-n', '--label', str(uuid.uuid4())] + + if request.param == 'core.code.installed': + options.extend(['--computer', aiida_localhost.pk, '--filepath-executable', '/usr/bin/bash']) + + if request.param == 'core.code.portable': + filepath_executable = 'bash' + (tmp_path / filepath_executable).touch() + options.extend(['--filepath-executable', filepath_executable, '--filepath-files', tmp_path]) + + return options, request.param + + +@pytest.mark.usefixtures('aiida_profile') +@pytest.mark.parametrize('command_options', ('core.code.installed', 'core.code.portable'), indirect=True) +def test_code_create(run_cli_command, command_options): + """Test the ``verdi code create`` command.""" + options, entry_point = command_options + cls = DataFactory(entry_point) + result = run_cli_command(cmd_code.code_create, options) + assert f'Success: Created {cls.__name__}' in result.output From 0f3dc5c65ca0dbd032e8d1cb0d34e6f7c107e417 Mon Sep 17 00:00:00 2001 From: Sebastiaan Huber Date: Sun, 1 May 2022 22:15:25 +0200 Subject: [PATCH 5/7] Remove use of legacy `Code` The `Code` plugin is used a lot and so cannot just be removed. Instead, we keep it for now and have it subclass the new `AbstractCode`. This way, the class already has the interface that it will have once the deprecated interface is removed. This will allow users to start updating their use of it in a backwards-compatible manner. The codebase is updated to replace the use of the deprecated interface with that of the new interface. No new instances of `Code` are created, only instances of the new `RemoteCode` and `LocalCode`. When the time comes to drop the legacy `Code` class, all that is needed is a data migration that will update existing instances to either an instance of `InstalledCode` or `PortableCode` depending on the attributes that the instance defines. The legacy methods and properties of `Code` plugin are deprecated. --- .../system_tests/pytest/test_memory_leaks.py | 6 +- aiida/cmdline/commands/cmd_code.py | 42 ++--- aiida/cmdline/params/types/code.py | 2 +- aiida/engine/daemon/execmanager.py | 6 +- aiida/engine/processes/calcjobs/calcjob.py | 22 +-- aiida/engine/processes/process.py | 4 +- aiida/manage/tests/pytest_fixtures.py | 23 ++- aiida/orm/nodes/data/code/legacy.py | 159 ++++++++++-------- aiida/orm/utils/builders/code.py | 44 +++-- aiida/workflows/arithmetic/multiply_add.py | 4 +- .../include/snippets/extend_workflows.py | 8 +- .../howto/include/snippets/plugins/launch.py | 2 +- docs/source/howto/plugin_codes.rst | 2 +- docs/source/howto/workchains_restart.rst | 2 +- docs/source/intro/tutorial.md | 4 +- tests/benchmark/test_engine.py | 10 +- tests/calculations/arithmetic/test_add.py | 8 +- tests/calculations/test_templatereplacer.py | 4 +- tests/cmdline/commands/test_archive_create.py | 10 +- tests/cmdline/commands/test_calcjob.py | 2 +- tests/cmdline/commands/test_code.py | 82 +++++---- tests/cmdline/commands/test_run.py | 12 +- tests/cmdline/params/types/test_code.py | 16 +- tests/conftest.py | 15 ++ .../processes/calcjobs/test_calc_job.py | 19 ++- tests/engine/processes/test_builder.py | 12 +- tests/engine/test_launch.py | 15 +- tests/engine/test_process.py | 7 +- tests/orm/data/test_code.py | 6 +- tests/orm/utils/test_loaders.py | 7 +- tests/test_generic.py | 12 +- tests/test_nodes.py | 23 +-- .../archive/migration/test_prov_redesign.py | 5 +- tests/tools/archive/orm/test_codes.py | 18 +- 34 files changed, 316 insertions(+), 297 deletions(-) diff --git a/.github/system_tests/pytest/test_memory_leaks.py b/.github/system_tests/pytest/test_memory_leaks.py index 4632d9bc3b..396793346f 100644 --- a/.github/system_tests/pytest/test_memory_leaks.py +++ b/.github/system_tests/pytest/test_memory_leaks.py @@ -50,8 +50,10 @@ def test_leak_ssh_calcjob(): Note: This relies on the 'slurm-ssh' computer being set up. """ - code = orm.Code( - input_plugin_name='core.arithmetic.add', remote_computer_exec=[orm.load_computer('slurm-ssh'), '/bin/bash'] + code = orm.InstalledCode( + default_calc_job_plugin='core.arithmetic.add', + computer=orm.load_computer('slurm-ssh'), + filepath_executable='/bin/bash' ) inputs = {'x': orm.Int(1), 'y': orm.Int(2), 'code': code} run_finished_ok(ArithmeticAddCalculation, **inputs) diff --git a/aiida/cmdline/commands/cmd_code.py b/aiida/cmdline/commands/cmd_code.py index e83edd0a5c..a7f1d0a5b1 100644 --- a/aiida/cmdline/commands/cmd_code.py +++ b/aiida/cmdline/commands/cmd_code.py @@ -112,7 +112,6 @@ def setup_code(ctx, non_interactive, **kwargs): except Exception as exception: # pylint: disable=broad-except echo.echo_critical(f'Unable to store the Code: {exception}') - code.reveal() echo.echo_success(f'Code<{code.pk}> {code.full_label} created') @@ -127,9 +126,11 @@ def code_test(code): * Whether the remote executable exists. """ - if not code.is_local(): + from aiida.orm import InstalledCode + + if isinstance(code, InstalledCode): try: - code.validate_remote_exec_path() + code.validate_filepath_executable() except exceptions.ValidationError as exception: echo.echo_critical(f'validation failed: {exception}') @@ -169,10 +170,11 @@ def code_duplicate(ctx, code, non_interactive, **kwargs): kwargs['code_type'] = CodeBuilder.CodeType.STORE_AND_UPLOAD if kwargs.pop('hide_original'): - code.hide() + code.is_hidden = True # Convert entry point to its name - kwargs['input_plugin'] = kwargs['input_plugin'].name + if kwargs['input_plugin']: + kwargs['input_plugin'] = kwargs['input_plugin'].name code_builder = ctx.code_builder for key, value in kwargs.items(): @@ -181,7 +183,7 @@ def code_duplicate(ctx, code, non_interactive, **kwargs): try: new_code.store() - new_code.reveal() + new_code.is_hidden = False except ValidationError as exception: echo.echo_critical(f'Unable to store the Code: {exception}') @@ -194,31 +196,15 @@ def code_duplicate(ctx, code, non_interactive, **kwargs): def show(code): """Display detailed information for a code.""" from aiida.cmdline import is_verbose - from aiida.repository import FileType table = [] table.append(['PK', code.pk]) table.append(['UUID', code.uuid]) table.append(['Label', code.label]) table.append(['Description', code.description]) - table.append(['Default plugin', code.get_input_plugin_name()]) - - if code.is_local(): - table.append(['Type', 'local']) - table.append(['Exec name', code.get_execname()]) - table.append(['List of files/folders:', '']) - for obj in code.list_objects(): - if obj.file_type == FileType.DIRECTORY: - table.append(['directory', obj.name]) - else: - table.append(['file', obj.name]) - else: - table.append(['Type', 'remote']) - table.append(['Remote machine', code.get_remote_computer().label]) - table.append(['Remote absolute path', code.get_remote_exec_path()]) - - table.append(['Prepend text', code.get_prepend_text()]) - table.append(['Append text', code.get_append_text()]) + table.append(['Default plugin', code.default_calc_job_plugin]) + table.append(['Prepend text', code.prepend_text]) + table.append(['Append text', code.append_text]) if is_verbose(): table.append(['Calculations', len(code.base.links.get_outgoing().all())]) @@ -258,7 +244,7 @@ def _dry_run_callback(pks): def hide(codes): """Hide one or more codes from `verdi code list`.""" for code in codes: - code.hide() + code.is_hidden = True echo.echo_success(f'Code<{code.pk}> {code.full_label} hidden') @@ -268,7 +254,7 @@ def hide(codes): def reveal(codes): """Reveal one or more hidden codes in `verdi code list`.""" for code in codes: - code.reveal() + code.is_hidden = False echo.echo_success(f'Code<{code.pk}> {code.full_label} revealed') @@ -281,7 +267,7 @@ def relabel(code, label): old_label = code.full_label try: - code.relabel(label) + code.label = label except (TypeError, ValueError) as exception: echo.echo_critical(f'invalid code label: {exception}') else: diff --git a/aiida/cmdline/params/types/code.py b/aiida/cmdline/params/types/code.py index 0ecd92b3f3..32367323ff 100644 --- a/aiida/cmdline/params/types/code.py +++ b/aiida/cmdline/params/types/code.py @@ -59,7 +59,7 @@ def convert(self, value, param, ctx): code = super().convert(value, param, ctx) if code and self._entry_point is not None: - entry_point = code.get_input_plugin_name() + entry_point = code.default_calc_job_plugin if entry_point != self._entry_point: raise click.BadParameter( 'the retrieved Code<{}> has plugin type "{}" while "{}" is required'.format( diff --git a/aiida/engine/daemon/execmanager.py b/aiida/engine/daemon/execmanager.py index 2a87c5e6e2..ab4236d70b 100644 --- a/aiida/engine/daemon/execmanager.py +++ b/aiida/engine/daemon/execmanager.py @@ -28,7 +28,7 @@ from aiida.common.folders import SandboxFolder from aiida.common.links import LinkType from aiida.manage.configuration import get_config_option -from aiida.orm import CalcJobNode, Code, FolderData, Node, RemoteData, load_node +from aiida.orm import CalcJobNode, Code, FolderData, Node, PortableCode, RemoteData, load_node from aiida.orm.utils.log import get_dblogger_extra from aiida.repository.common import FileType from aiida.schedulers.datastructures import JobState @@ -174,7 +174,7 @@ def upload_calculation( # Still, beware! The code file itself could be overwritten... # But I checked for this earlier. for code in input_codes: - if code.is_local(): + if isinstance(code, PortableCode): # Note: this will possibly overwrite files for filename in code.base.repository.list_object_names(): # Note, once #2579 is implemented, use the `node.open` method instead of the named temporary file in @@ -184,7 +184,7 @@ def upload_calculation( handle.write(code.base.repository.get_object_content(filename, mode='rb')) handle.flush() transport.put(handle.name, filename) - transport.chmod(code.get_local_executable(), 0o755) # rwxr-xr-x + transport.chmod(code.filepath_executable, 0o755) # rwxr-xr-x # local_copy_list is a list of tuples, each with (uuid, dest_path, rel_path) # NOTE: validation of these lists are done inside calculation.presubmit() diff --git a/aiida/engine/processes/calcjobs/calcjob.py b/aiida/engine/processes/calcjobs/calcjob.py index 9d71583374..3b07181678 100644 --- a/aiida/engine/processes/calcjobs/calcjob.py +++ b/aiida/engine/processes/calcjobs/calcjob.py @@ -194,7 +194,7 @@ def define(cls, spec: CalcJobProcessSpec) -> None: # type: ignore[override] # yapf: disable super().define(spec) spec.inputs.validator = validate_calc_job # type: ignore[assignment] # takes only PortNamespace not Port - spec.input('code', valid_type=orm.Code, required=False, + spec.input('code', valid_type=orm.AbstractCode, required=False, help='The `Code` to use for this job. This input is required, unless the `remote_folder` input is ' 'specified, which means an existing job is being imported and no code will actually be run.') spec.input('remote_folder', valid_type=orm.RemoteData, required=False, @@ -592,7 +592,7 @@ def presubmit(self, folder: Folder) -> CalcInfo: from aiida.common.datastructures import CodeInfo, CodeRunMode from aiida.common.exceptions import InputValidationError, InvalidOperation, PluginInternalError, ValidationError from aiida.common.utils import validate_list_of_string_tuples - from aiida.orm import Code, Computer, load_node + from aiida.orm import AbstractCode, Code, Computer, InstalledCode, PortableCode, load_node from aiida.schedulers.datastructures import JobTemplate, JobTemplateCodeInfo inputs = self.node.base.links.get_incoming(link_type=LinkType.INPUT_CALC) @@ -601,19 +601,19 @@ def presubmit(self, folder: Folder) -> CalcInfo: raise InvalidOperation('calculation node is not stored.') computer = self.node.computer - codes = [_ for _ in inputs.all_nodes() if isinstance(_, Code)] + codes = [_ for _ in inputs.all_nodes() if isinstance(_, AbstractCode)] for code in codes: - if not code.can_run_on(computer): + if not code.can_run_on_computer(computer): raise InputValidationError( 'The selected code {} for calculation {} cannot run on computer {}'.format( code.pk, self.node.pk, computer.label ) ) - if code.is_local() and code.get_local_executable() in folder.get_content_list(): + if isinstance(code, PortableCode) and str(code.filepath_executable) in folder.get_content_list(): raise PluginInternalError( - f'The plugin created a file {code.get_local_executable()} that is also the executable name!' + f'The plugin created a file {code.filepath_executable} that is also the executable name!' ) calc_info = self.prepare_for_submission(folder) @@ -663,12 +663,12 @@ def presubmit(self, folder: Folder) -> CalcInfo: # would return None, in which case the join method would raise # an exception prepend_texts = [computer.get_prepend_text()] + \ - [code.get_prepend_text() for code in codes] + \ + [code.prepend_text for code in codes] + \ [calc_info.prepend_text, self.node.get_option('prepend_text')] job_tmpl.prepend_text = '\n\n'.join(prepend_text for prepend_text in prepend_texts if prepend_text) append_texts = [self.node.get_option('append_text'), calc_info.append_text] + \ - [code.get_append_text() for code in codes] + \ + [code.append_text for code in codes] + \ [computer.get_append_text()] job_tmpl.append_text = '\n\n'.join(append_text for append_text in append_texts if append_text) @@ -696,7 +696,7 @@ def presubmit(self, folder: Folder) -> CalcInfo: if code_info.code_uuid is None: raise PluginInternalError('CalcInfo should have the information of the code to be launched') - this_code = load_node(code_info.code_uuid, sub_classes=(Code,)) + this_code = load_node(code_info.code_uuid, sub_classes=(Code, InstalledCode, PortableCode)) # To determine whether this code should be run with MPI enabled, we get the value that was set in the inputs # of the entire process, which can then be overwritten by the value from the `CodeInfo`. This allows plugins @@ -715,12 +715,12 @@ def presubmit(self, folder: Folder) -> CalcInfo: else: prepend_cmdline_params = [] - cmdline_params = [this_code.get_execname()] + (code_info.cmdline_params or []) + cmdline_params = [str(this_code.get_executable())] + (code_info.cmdline_params or []) tmpl_code_info = JobTemplateCodeInfo() tmpl_code_info.prepend_cmdline_params = prepend_cmdline_params tmpl_code_info.cmdline_params = cmdline_params - tmpl_code_info.use_double_quotes = [computer.get_use_double_quotes(), this_code.get_use_double_quotes()] + tmpl_code_info.use_double_quotes = [computer.get_use_double_quotes(), this_code.use_double_quotes] tmpl_code_info.stdin_name = code_info.stdin_name tmpl_code_info.stdout_name = code_info.stdout_name tmpl_code_info.stderr_name = code_info.stderr_name diff --git a/aiida/engine/processes/process.py b/aiida/engine/processes/process.py index 8b98ec6a9d..cbe20755b7 100644 --- a/aiida/engine/processes/process.py +++ b/aiida/engine/processes/process.py @@ -713,8 +713,8 @@ def _setup_inputs(self) -> None: continue # Special exception: set computer if node is a remote Code and our node does not yet have a computer set - if isinstance(node, orm.Code) and not node.is_local() and not self.node.computer: - self.node.computer = node.get_remote_computer() + if isinstance(node, orm.InstalledCode) and not self.node.computer: + self.node.computer = node.computer # Need this special case for tests that use ProcessNodes as classes if isinstance(self.node, orm.CalculationNode): diff --git a/aiida/manage/tests/pytest_fixtures.py b/aiida/manage/tests/pytest_fixtures.py index 9173ff67e0..b4ea2fba11 100644 --- a/aiida/manage/tests/pytest_fixtures.py +++ b/aiida/manage/tests/pytest_fixtures.py @@ -199,13 +199,18 @@ def get_code(entry_point, executable, computer=aiida_localhost, label=None, prep :rtype: :py:class:`aiida.orm.Code` """ from aiida.common import exceptions - from aiida.orm import Code, Computer, QueryBuilder + from aiida.orm import Computer, InstalledCode, QueryBuilder if label is None: label = executable builder = QueryBuilder().append(Computer, filters={'uuid': computer.uuid}, tag='computer') - builder.append(Code, filters={'label': label, 'attributes.input_plugin': entry_point}, with_computer='computer') + builder.append( + InstalledCode, filters={ + 'label': label, + 'attributes.input_plugin': entry_point + }, with_computer='computer' + ) try: code = builder.one()[0] @@ -218,15 +223,19 @@ def get_code(entry_point, executable, computer=aiida_localhost, label=None, prep if not executable_path: raise ValueError(f'The executable "{executable}" was not found in the $PATH.') - code = Code(input_plugin_name=entry_point, remote_computer_exec=[computer, executable_path]) - code.label = label - code.description = label + code = InstalledCode( + label=label, + description=label, + default_calc_job_plugin=entry_point, + computer=computer, + filepath_executable=executable_path + ) if prepend_text is not None: - code.set_prepend_text(prepend_text) + code.prepend_text = prepend_text if append_text is not None: - code.set_append_text(append_text) + code.append_text = append_text return code.store() diff --git a/aiida/orm/nodes/data/code/legacy.py b/aiida/orm/nodes/data/code/legacy.py index 445b0bc64a..4f8961fabb 100644 --- a/aiida/orm/nodes/data/code/legacy.py +++ b/aiida/orm/nodes/data/code/legacy.py @@ -12,13 +12,15 @@ from aiida.common import exceptions from aiida.common.log import override_log_level +from aiida.common.warnings import warn_deprecation +from aiida.orm import Computer -from ..data import Data +from .abstract import AbstractCode __all__ = ('Code',) -class Code(Data): +class Code(AbstractCode): """ A code entity. It can either be 'local', or 'remote'. @@ -44,9 +46,13 @@ def __init__(self, remote_computer_exec=None, local_executable=None, input_plugi raise ValueError('cannot set `remote_computer_exec` and `local_executable` at the same time') if remote_computer_exec: + warn_deprecation( + 'The `Code` plugin is deprecated, use the `InstalledCode` (`core.code.remote`) instead.', 3 + ) self.set_remote_computer_exec(remote_computer_exec) if local_executable: + warn_deprecation('The `Code` plugin is deprecated, use the `PortableCode` (`core.code.local`) instead.', 3) self.set_local_executable(local_executable) if input_plugin_name: @@ -57,25 +63,53 @@ def __init__(self, remote_computer_exec=None, local_executable=None, input_plugi HIDDEN_KEY = 'hidden' + def can_run_on_computer(self, computer: Computer) -> bool: + """Return whether the code can run on a given computer. + + :param computer: The computer. + :return: ``True`` if the code can run on ``computer``, ``False`` otherwise. + """ + from aiida import orm + from aiida.common.lang import type_check + + if self.is_local(): + return True + + type_check(computer, orm.Computer) + return computer.pk == self.get_remote_computer().pk + + def get_executable(self) -> str: + """Return the executable that the submission script should execute to run the code. + + :return: The executable to be called in the submission script. + """ + if self.is_local(): + return f'./{self.get_local_executable()}' + + return self.get_remote_exec_path() + def hide(self): """ Hide the code (prevents from showing it in the verdi code list) """ - self.base.extras.set(self.HIDDEN_KEY, True) + warn_deprecation('This property is deprecated, use the `Code.is_hidden` property instead.', version=3) + self.is_hidden = True def reveal(self): """ Reveal the code (allows to show it in the verdi code list) By default, it is revealed """ - self.base.extras.set(self.HIDDEN_KEY, False) + warn_deprecation('This property is deprecated, use the `Code.is_hidden` property instead.', version=3) + self.is_hidden = False @property def hidden(self): """ Determines whether the Code is hidden or not """ - return self.base.extras.get(self.HIDDEN_KEY, False) + warn_deprecation('This property is deprecated, use the `Code.is_hidden` property instead.', version=3) + return self.is_hidden def set_files(self, files): """ @@ -102,6 +136,9 @@ def __str__(self): def get_computer_label(self): """Get label of this code's computer.""" + warn_deprecation( + 'This method is deprecated, use the `InstalledCode.computer.label` property instead.', version=3 + ) return 'repository' if self.is_local() else self.computer.label @property @@ -110,33 +147,14 @@ def full_label(self): Returns label of the form @. """ - return f'{self.label}@{self.get_computer_label()}' - - @property - def label(self): - """Return the node label. - - :return: the label - """ - return super().label - - @label.setter - def label(self, value): - """Set the label. - - :param value: the new value to set - """ - if '@' in str(value): - msg = "Code labels must not contain the '@' symbol" - raise ValueError(msg) - - super(Code, self.__class__).label.fset(self, value) # pylint: disable=no-member + return f'{self.label}@{"repository" if self.is_local() else self.computer.label}' def relabel(self, new_label): """Relabel this code. :param new_label: new code label """ + warn_deprecation('This method is deprecated, use the `label` property instead.', version=3) # pylint: disable=unused-argument suffix = f'@{self.computer.label}' if new_label.endswith(suffix): @@ -149,6 +167,7 @@ def get_description(self): :return: string description of this Code instance """ + warn_deprecation('This method is deprecated, use the `description` property instead.', version=3) return f'{self.description}' @classmethod @@ -162,9 +181,10 @@ def get_code_helper(cls, label, machinename=None, backend=None): a code """ from aiida.common.exceptions import MultipleObjectsError, NotExistent - from aiida.orm.computers import Computer from aiida.orm.querybuilder import QueryBuilder + warn_deprecation('This method is deprecated, use `aiida.orm.load_code` instead.', version=3) + query = QueryBuilder(backend=backend) query.append(cls, filters={'label': label}, project='*', tag='code') if machinename: @@ -198,6 +218,8 @@ def get(cls, pk=None, label=None, machinename=None): # pylint: disable=arguments-differ from aiida.orm.utils import load_code + warn_deprecation('This method is deprecated, use `aiida.orm.load_code` instead.', version=3) + # first check if code pk is provided if pk: code_int = int(pk) @@ -237,6 +259,8 @@ def get_from_string(cls, code_string): """ from aiida.common.exceptions import MultipleObjectsError, NotExistent + warn_deprecation('This method is deprecated, use `aiida.orm.load_code` instead.', version=3) + try: label, _, machinename = code_string.partition('@') except AttributeError: @@ -261,6 +285,9 @@ def list_for_plugin(cls, plugin, labels=True, backend=None): otherwise a list of integers with the code PKs. """ from aiida.orm.querybuilder import QueryBuilder + + warn_deprecation('This method has been deprecated, there is no replacement.', version=3) + query = QueryBuilder(backend=backend) query.append(cls, filters={'attributes.input_plugin': {'==': plugin}}) valid_codes = query.all(flat=True) @@ -304,6 +331,10 @@ def validate_remote_exec_path(self): :raises `~aiida.common.exceptions.ValidationError`: if no transport could be opened or if the defined executable does not exist on the remote computer. """ + warn_deprecation( + 'This method is deprecated, use the `InstalledCode.validate_filepath_executable` property instead.', + version=3 + ) filepath = self.get_remote_exec_path() try: @@ -325,71 +356,77 @@ def set_prepend_text(self, code): Pass a string of code that will be put in the scheduler script before the execution of the code. """ - self.base.attributes.set('prepend_text', str(code)) + warn_deprecation('This method is deprecated, use the `prepend_text` property instead.', version=3) + self.prepend_text = code def get_prepend_text(self): """ Return the code that will be put in the scheduler script before the execution, or an empty string if no pre-exec code was defined. """ - return self.base.attributes.get('prepend_text', '') + warn_deprecation('This method is deprecated, use the `prepend_text` property instead.', version=3) + return self.prepend_text def set_input_plugin_name(self, input_plugin): """ Set the name of the default input plugin, to be used for the automatic generation of a new calculation. """ - if input_plugin is None: - self.base.attributes.set('input_plugin', None) - else: - self.base.attributes.set('input_plugin', str(input_plugin)) + warn_deprecation('This method is deprecated, use the `default_calc_job_plugin` property instead.', version=3) + self.default_calc_job_plugin = input_plugin def get_input_plugin_name(self): """ Return the name of the default input plugin (or None if no input plugin was set. """ - return self.base.attributes.get('input_plugin', None) + warn_deprecation('This method is deprecated, use the `default_calc_job_plugin` property instead.', version=3) + return self.default_calc_job_plugin def set_use_double_quotes(self, use_double_quotes: bool): """Set whether the command line invocation of this code should be escaped with double quotes. :param use_double_quotes: True if to escape with double quotes, False otherwise. """ - from aiida.common.lang import type_check - type_check(use_double_quotes, bool) - self.base.attributes.set('use_double_quotes', use_double_quotes) + warn_deprecation('This method is deprecated, use the `use_double_quotes` property instead.', version=3) + self.use_double_quotes = use_double_quotes def get_use_double_quotes(self) -> bool: """Return whether the command line invocation of this code should be escaped with double quotes. :returns: True if to escape with double quotes, False otherwise which is also the default. """ - return self.base.attributes.get('use_double_quotes', False) + warn_deprecation('This method is deprecated, use the `use_double_quotes` property instead.', version=3) + return self.use_double_quotes def set_append_text(self, code): """ Pass a string of code that will be put in the scheduler script after the execution of the code. """ - self.base.attributes.set('append_text', str(code)) + warn_deprecation('This method is deprecated, use the `append_text` property instead.', version=3) + self.append_text = code def get_append_text(self): """ Return the postexec_code, or an empty string if no post-exec code was defined. """ - return self.base.attributes.get('append_text', '') + warn_deprecation('This method is deprecated, use the `append_text` property instead.', version=3) + return self.append_text def set_local_executable(self, exec_name): """ Set the filename of the local executable. Implicitly set the code as local. """ + warn_deprecation('This method is deprecated, use `PortableCode`.', version=3) + self._set_local() - self.base.attributes.set('local_executable', exec_name) + self.filepath_executable = exec_name def get_local_executable(self): - return self.base.attributes.get('local_executable', '') + warn_deprecation('This method is deprecated, use `PortableCode.filepath_executable` instead.', version=3) + return self.filepath_executable def set_remote_computer_exec(self, remote_computer_exec): """ @@ -402,6 +439,8 @@ def set_remote_computer_exec(self, remote_computer_exec): from aiida import orm from aiida.common.lang import type_check + warn_deprecation('This method is deprecated, use `InstalledCode`.', version=3) + if (not isinstance(remote_computer_exec, (list, tuple)) or len(remote_computer_exec) != 2): raise ValueError( 'remote_computer_exec must be a list or tuple of length 2, with machine and executable name' @@ -419,11 +458,14 @@ def set_remote_computer_exec(self, remote_computer_exec): self.base.attributes.set('remote_exec_path', remote_exec_path) def get_remote_exec_path(self): + warn_deprecation('This method is deprecated, use `InstalledCode.filepath_executable` instead.', version=3) if self.is_local(): raise ValueError('The code is local') return self.base.attributes.get('remote_exec_path', '') def get_remote_computer(self): + """Return the remote computer associated with this code.""" + warn_deprecation('This method is deprecated, use the `computer` attribute instead.', version=3) if self.is_local(): raise ValueError('The code is local') @@ -463,6 +505,7 @@ def is_local(self): Return True if the code is 'local', False if it is 'remote' (see also documentation of the set_local and set_remote functions). """ + warn_deprecation('This method is deprecated, use a `PortableCode` instance and check the type.', version=3) return self.base.attributes.get('is_local', None) def can_run_on(self, computer): @@ -477,6 +520,8 @@ def can_run_on(self, computer): from aiida import orm from aiida.common.lang import type_check + warn_deprecation('This method is deprecated, use `can_run_on_computer` instead.', version=3) + if self.is_local(): return True @@ -489,35 +534,9 @@ def get_execname(self): For local codes, it is ./LOCAL_EXECUTABLE_NAME For remote codes, it is the absolute path to the executable. """ + warn_deprecation('This method is deprecated, use `get_executable` instead.', version=3) + if self.is_local(): return f'./{self.get_local_executable()}' return self.get_remote_exec_path() - - def get_builder(self): - """Create and return a new `ProcessBuilder` for the `CalcJob` class of the plugin configured for this code. - - The configured calculation plugin class is defined by the `get_input_plugin_name` method. - - .. note:: it also sets the ``builder.code`` value. - - :return: a `ProcessBuilder` instance with the `code` input already populated with ourselves - :raise aiida.common.EntryPointError: if the specified plugin does not exist. - :raise ValueError: if no default plugin was specified. - """ - from aiida.plugins import CalculationFactory - - plugin_name = self.get_input_plugin_name() - - if plugin_name is None: - raise ValueError('no default calculation input plugin specified for this code') - - try: - process_class = CalculationFactory(plugin_name) - except exceptions.EntryPointError: - raise exceptions.EntryPointError(f'the calculation entry point `{plugin_name}` could not be loaded') - - builder = process_class.get_builder() - builder.code = self - - return builder diff --git a/aiida/orm/utils/builders/code.py b/aiida/orm/utils/builders/code.py index a583f06652..6028748fcc 100644 --- a/aiida/orm/utils/builders/code.py +++ b/aiida/orm/utils/builders/code.py @@ -9,16 +9,18 @@ ########################################################################### """Manage code objects with lazy loading of the db env""" import enum -import os +import pathlib from aiida.cmdline.utils.decorators import with_dbenv from aiida.common.utils import ErrorAccumulator +from aiida.orm import InstalledCode, PortableCode class CodeBuilder: """Build a code with validation of attribute combinations""" def __init__(self, **kwargs): + """Construct a new instance.""" self._err_acc = ErrorAccumulator(self.CodeValidationError) self._code_spec = {} @@ -41,31 +43,27 @@ def new(self): """Build and return a new code instance (not stored)""" self.validate() - from aiida.orm import Code - # Will be used at the end to check if all keys are known (those that are not None) passed_keys = set(k for k in self._code_spec.keys() if self._code_spec[k] is not None) used = set() if self._get_and_count('code_type', used) == self.CodeType.STORE_AND_UPLOAD: - file_list = [ - os.path.realpath(os.path.join(self.code_folder, f)) - for f in os.listdir(self._get_and_count('code_folder', used)) - ] - code = Code(local_executable=self._get_and_count('code_rel_path', used), files=file_list) + code = PortableCode( + filepath_executable=self._get_and_count('code_rel_path', used), + filepath_files=pathlib.Path(self._get_and_count('code_folder', used)) + ) else: - code = Code( - remote_computer_exec=( - self._get_and_count('computer', used), self._get_and_count('remote_abs_path', used) - ) + code = InstalledCode( + computer=self._get_and_count('computer', used), + filepath_executable=self._get_and_count('remote_abs_path', used) ) code.label = self._get_and_count('label', used) code.description = self._get_and_count('description', used) - code.set_input_plugin_name(self._get_and_count('input_plugin', used)) - code.set_use_double_quotes(self._get_and_count('use_double_quotes', used)) - code.set_prepend_text(self._get_and_count('prepend_text', used)) - code.set_append_text(self._get_and_count('append_text', used)) + code.default_calc_job_plugin = self._get_and_count('input_plugin', used) + code.use_double_quotes = self._get_and_count('use_double_quotes', used) + code.prepend_text = self._get_and_count('prepend_text', used) + code.append_text = self._get_and_count('append_text', used) # Complain if there are keys that are passed but not used if passed_keys - used: @@ -98,19 +96,19 @@ def get_code_spec(code): spec = {} spec['label'] = code.label spec['description'] = code.description - spec['input_plugin'] = code.get_input_plugin_name() - spec['use_double_quotes'] = code.get_use_double_quotes() - spec['prepend_text'] = code.get_prepend_text() - spec['append_text'] = code.get_append_text() + spec['input_plugin'] = code.default_calc_job_plugin + spec['use_double_quotes'] = code.use_double_quotes + spec['prepend_text'] = code.prepend_text + spec['append_text'] = code.append_text - if code.is_local(): + if isinstance(code, PortableCode): spec['code_type'] = CodeBuilder.CodeType.STORE_AND_UPLOAD spec['code_folder'] = code.get_code_folder() spec['code_rel_path'] = code.get_code_rel_path() else: spec['code_type'] = CodeBuilder.CodeType.ON_COMPUTER - spec['computer'] = code.get_remote_computer() - spec['remote_abs_path'] = code.get_remote_exec_path() + spec['computer'] = code.computer + spec['remote_abs_path'] = code.get_executable() return spec diff --git a/aiida/workflows/arithmetic/multiply_add.py b/aiida/workflows/arithmetic/multiply_add.py index 858979816f..3576162ce1 100644 --- a/aiida/workflows/arithmetic/multiply_add.py +++ b/aiida/workflows/arithmetic/multiply_add.py @@ -11,7 +11,7 @@ # start-marker for docs """Implementation of the MultiplyAddWorkChain for testing and demonstration purposes.""" from aiida.engine import ToContext, WorkChain, calcfunction -from aiida.orm import Code, Int +from aiida.orm import AbstractCode, Int from aiida.plugins.factories import CalculationFactory ArithmeticAddCalculation = CalculationFactory('core.arithmetic.add') @@ -32,7 +32,7 @@ def define(cls, spec): spec.input('x', valid_type=Int) spec.input('y', valid_type=Int) spec.input('z', valid_type=Int) - spec.input('code', valid_type=Code) + spec.input('code', valid_type=AbstractCode) spec.outline( cls.multiply, cls.add, diff --git a/docs/source/howto/include/snippets/extend_workflows.py b/docs/source/howto/include/snippets/extend_workflows.py index 7975d0ae8e..568ed51e60 100644 --- a/docs/source/howto/include/snippets/extend_workflows.py +++ b/docs/source/howto/include/snippets/extend_workflows.py @@ -11,7 +11,7 @@ # start-marker for docs """Code snippets for the "How to extend workflows" section.""" from aiida.engine import ToContext, WorkChain, calcfunction -from aiida.orm import Bool, Code, Int +from aiida.orm import AbstractCode, Bool, Int from aiida.plugins.factories import CalculationFactory ArithmeticAddCalculation = CalculationFactory('core.arithmetic.add') @@ -38,7 +38,7 @@ def define(cls, spec): spec.input('x', valid_type=Int) spec.input('y', valid_type=Int) spec.input('z', valid_type=Int) - spec.input('code', valid_type=Code) + spec.input('code', valid_type=AbstractCode) spec.outline( cls.multiply, cls.add, @@ -81,7 +81,7 @@ def define(cls, spec): spec.input('x', valid_type=Int) spec.input('y', valid_type=Int) spec.input('z', valid_type=Int) - spec.input('code', valid_type=Code) + spec.input('code', valid_type=AbstractCode) spec.outline( cls.multiply, cls.add, @@ -126,7 +126,7 @@ def define(cls, spec): spec.input('x', valid_type=Int) spec.input('y', valid_type=Int) spec.input('z', valid_type=Int) - spec.input('code', valid_type=Code) + spec.input('code', valid_type=AbstractCode) spec.outline( cls.multiply_add, cls.is_even, diff --git a/docs/source/howto/include/snippets/plugins/launch.py b/docs/source/howto/include/snippets/plugins/launch.py index 3c17f2aa14..f519047b95 100644 --- a/docs/source/howto/include/snippets/plugins/launch.py +++ b/docs/source/howto/include/snippets/plugins/launch.py @@ -13,7 +13,7 @@ code = orm.load_code('diff@localhost') except NotExistent: # Setting up code via python API (or use "verdi code setup") - code = orm.Code(label='diff', remote_computer_exec=[computer, '/usr/bin/diff'], input_plugin_name='diff-tutorial') + code = orm.InstalledCode(label='diff', computer=computer, filepath_executable='/usr/bin/diff', default_calc_job_plugin='diff-tutorial') # Set up inputs builder = code.get_builder() diff --git a/docs/source/howto/plugin_codes.rst b/docs/source/howto/plugin_codes.rst index 32c8b0c7a7..d899620890 100644 --- a/docs/source/howto/plugin_codes.rst +++ b/docs/source/howto/plugin_codes.rst @@ -125,7 +125,7 @@ There is no ``return`` statement in ``define``: the ``define`` method directly m .. code-block:: python - spec.input('code', valid_type=orm.Code, help='The `Code` to use for this job.') + spec.input('code', valid_type=orm.AbstractCode, help='The `Code` to use for this job.') .. admonition:: Further reading diff --git a/docs/source/howto/workchains_restart.rst b/docs/source/howto/workchains_restart.rst index 0ca636d323..9dda7afdfb 100644 --- a/docs/source/howto/workchains_restart.rst +++ b/docs/source/howto/workchains_restart.rst @@ -107,7 +107,7 @@ Next, as with all work chains, we should *define* its process specification: super().define(spec) spec.input('x', valid_type=(orm.Int, orm.Float), help='The left operand.') spec.input('y', valid_type=(orm.Int, orm.Float), help='The right operand.') - spec.input('code', valid_type=orm.Code, help='The code to use to perform the summation.') + spec.input('code', valid_type=orm.AbstractCode, help='The code to use to perform the summation.') spec.output('sum', valid_type=(orm.Int, orm.Float), help='The sum of the left and right operand.') spec.outline( cls.setup, diff --git a/docs/source/intro/tutorial.md b/docs/source/intro/tutorial.md index 32d9bcb278..592b74b866 100644 --- a/docs/source/intro/tutorial.md +++ b/docs/source/intro/tutorial.md @@ -443,7 +443,7 @@ These steps are provided as methods of the `MultiplyAddWorkChain` class. :tags: ["hide-cell"] from aiida.engine import ToContext, WorkChain, calcfunction -from aiida.orm import Code, Int +from aiida.orm import AbstractCode, Int from aiida.plugins.factories import CalculationFactory ArithmeticAddCalculation = CalculationFactory('core.arithmetic.add') @@ -464,7 +464,7 @@ class MultiplyAddWorkChain(WorkChain): spec.input('x', valid_type=Int) spec.input('y', valid_type=Int) spec.input('z', valid_type=Int) - spec.input('code', valid_type=Code) + spec.input('code', valid_type=AbstractCode) spec.outline( cls.multiply, cls.add, diff --git a/tests/benchmark/test_engine.py b/tests/benchmark/test_engine.py index e4cc385033..a99438d565 100644 --- a/tests/benchmark/test_engine.py +++ b/tests/benchmark/test_engine.py @@ -19,7 +19,7 @@ from aiida.engine import WorkChain, run_get_node, submit, while_ from aiida.manage import get_manager -from aiida.orm import Code, Int +from aiida.orm import InstalledCode, Int from aiida.plugins.factories import CalculationFactory ArithmeticAddCalculation = CalculationFactory('core.arithmetic.add') @@ -119,7 +119,9 @@ def run_task(self): @pytest.mark.benchmark(group='engine') def test_workchain_local(benchmark, aiida_localhost, workchain, iterations, outgoing): """Benchmark Workchains, executed in the local runner.""" - code = Code(input_plugin_name='core.arithmetic.add', remote_computer_exec=[aiida_localhost, '/bin/true']) + code = InstalledCode( + default_calc_job_plugin='core.arithmetic.add', computer=aiida_localhost, filepath_executable='/bin/true' + ) def _run(): return run_get_node(workchain, iterations=Int(iterations), code=code) @@ -175,7 +177,9 @@ async def _do_submit(): @pytest.mark.benchmark(group='engine') def test_workchain_daemon(benchmark, submit_get_node, aiida_localhost, workchain, iterations, outgoing): """Benchmark Workchains, executed in the via a daemon runner.""" - code = Code(input_plugin_name='core.arithmetic.add', remote_computer_exec=[aiida_localhost, '/bin/true']) + code = InstalledCode( + default_calc_job_plugin='core.arithmetic.add', computer=aiida_localhost, filepath_executable='/bin/true' + ) def _run(): return submit_get_node(workchain, iterations=Int(iterations), code=code) diff --git a/tests/calculations/arithmetic/test_add.py b/tests/calculations/arithmetic/test_add.py index a78cc45000..fac81f1556 100644 --- a/tests/calculations/arithmetic/test_add.py +++ b/tests/calculations/arithmetic/test_add.py @@ -20,7 +20,11 @@ def test_add_default(fixture_sandbox, aiida_localhost, generate_calc_job): """Test a default `ArithmeticAddCalculation`.""" entry_point_name = 'core.arithmetic.add' - inputs = {'x': orm.Int(1), 'y': orm.Int(2), 'code': orm.Code(remote_computer_exec=(aiida_localhost, '/bin/bash'))} + inputs = { + 'x': orm.Int(1), + 'y': orm.Int(2), + 'code': orm.InstalledCode(computer=aiida_localhost, filepath_executable='/bin/bash') + } calc_info = generate_calc_job(fixture_sandbox, entry_point_name, inputs) options = ArithmeticAddCalculation.spec().inputs['metadata']['options'] @@ -54,7 +58,7 @@ def test_add_custom_filenames(fixture_sandbox, aiida_localhost, generate_calc_jo inputs = { 'x': orm.Int(1), 'y': orm.Int(2), - 'code': orm.Code(remote_computer_exec=(aiida_localhost, '/bin/bash')), + 'code': orm.InstalledCode(computer=aiida_localhost, filepath_executable='/bin/bash'), 'metadata': { 'options': { 'input_filename': input_filename, diff --git a/tests/calculations/test_templatereplacer.py b/tests/calculations/test_templatereplacer.py index 6ed859d20c..3dbaaf23f4 100644 --- a/tests/calculations/test_templatereplacer.py +++ b/tests/calculations/test_templatereplacer.py @@ -24,7 +24,7 @@ def test_base_template(fixture_sandbox, aiida_localhost, generate_calc_job): entry_point_name = 'core.templatereplacer' inputs = { 'code': - orm.Code(remote_computer_exec=(aiida_localhost, '/bin/bash')), + orm.InstalledCode(computer=aiida_localhost, filepath_executable='/bin/bash'), 'metadata': { 'options': { 'resources': { @@ -83,7 +83,7 @@ def test_file_usage(fixture_sandbox, aiida_localhost, generate_calc_job): # Check that the files are correctly copied to the copy list entry_point_name = 'core.templatereplacer' inputs = { - 'code': orm.Code(remote_computer_exec=(aiida_localhost, '/bin/bash')), + 'code': orm.InstalledCode(computer=aiida_localhost, filepath_executable='/bin/bash'), 'metadata': { 'options': { 'resources': { diff --git a/tests/cmdline/commands/test_archive_create.py b/tests/cmdline/commands/test_archive_create.py index ee8fd60205..558646fbf7 100644 --- a/tests/cmdline/commands/test_archive_create.py +++ b/tests/cmdline/commands/test_archive_create.py @@ -14,7 +14,7 @@ import pytest from aiida.cmdline.commands import cmd_archive -from aiida.orm import Code, Computer, Dict, Group +from aiida.orm import Computer, Dict, Group, InstalledCode from aiida.storage.sqlite_zip.migrator import list_versions from aiida.tools.archive import ArchiveFormatSqlZip from tests.utils.archives import get_archive_file @@ -46,7 +46,7 @@ def test_create_all(run_cli_command, tmp_path): scheduler_type='core.direct', workdir='/tmp/aiida' ).store() - code = Code(remote_computer_exec=(computer, '/bin/true')).store() + code = InstalledCode(computer=computer, filepath_executable='/bin/true').store() group = Group(label='test_group').store() filename_output = tmp_path / 'archive.aiida' @@ -56,7 +56,7 @@ def test_create_all(run_cli_command, tmp_path): assert ArchiveFormatSqlZip().read_version(filename_output) == ArchiveFormatSqlZip().latest_version with ArchiveFormatSqlZip().open(filename_output, 'r') as archive: assert archive.querybuilder().append(Computer, project=['uuid']).all(flat=True) == [computer.uuid] - assert archive.querybuilder().append(Code, project=['uuid']).all(flat=True) == [code.uuid] + assert archive.querybuilder().append(InstalledCode, project=['uuid']).all(flat=True) == [code.uuid] assert archive.querybuilder().append(Group, project=['uuid']).all(flat=True) == [group.uuid] @@ -70,7 +70,7 @@ def test_create_basic(run_cli_command, tmp_path): scheduler_type='core.direct', workdir='/tmp/aiida' ).store() - code = Code(remote_computer_exec=(computer, '/bin/true')).store() + code = InstalledCode(computer=computer, filepath_executable='/bin/true').store() group = Group(label='test_group').store() node = Dict().store() filename_output = tmp_path / 'archive.aiida' @@ -81,7 +81,7 @@ def test_create_basic(run_cli_command, tmp_path): assert ArchiveFormatSqlZip().read_version(filename_output) == ArchiveFormatSqlZip().latest_version with ArchiveFormatSqlZip().open(filename_output, 'r') as archive: assert archive.querybuilder().append(Computer, project=['uuid']).all(flat=True) == [computer.uuid] - assert archive.querybuilder().append(Code, project=['uuid']).all(flat=True) == [code.uuid] + assert archive.querybuilder().append(InstalledCode, project=['uuid']).all(flat=True) == [code.uuid] assert archive.querybuilder().append(Group, project=['uuid']).all(flat=True) == [group.uuid] assert archive.querybuilder().append(Dict, project=['uuid']).all(flat=True) == [node.uuid] diff --git a/tests/cmdline/commands/test_calcjob.py b/tests/cmdline/commands/test_calcjob.py index ff619c7ec1..4dc5040f16 100644 --- a/tests/cmdline/commands/test_calcjob.py +++ b/tests/cmdline/commands/test_calcjob.py @@ -38,7 +38,7 @@ def init_profile(self, aiida_profile_clean, aiida_localhost): # pylint: disable # pylint: disable=attribute-defined-outside-init self.computer = aiida_localhost - self.code = orm.Code(remote_computer_exec=(self.computer, '/bin/true')).store() + self.code = orm.InstalledCode(computer=self.computer, filepath_executable='/bin/true').store() self.group = orm.Group(label='test_group').store() self.node = orm.Data().store() self.calcs = [] diff --git a/tests/cmdline/commands/test_code.py b/tests/cmdline/commands/test_code.py index 38898a93ba..b89a079389 100644 --- a/tests/cmdline/commands/test_code.py +++ b/tests/cmdline/commands/test_code.py @@ -9,7 +9,9 @@ ########################################################################### # pylint: disable=unused-argument,redefined-outer-name """Tests for the 'verdi code' command.""" +import io import os +import pathlib import tempfile import textwrap import uuid @@ -20,21 +22,22 @@ from aiida.cmdline.commands import cmd_code from aiida.cmdline.params.options.commands.code import validate_label_uniqueness from aiida.common.exceptions import MultipleObjectsError, NotExistent -from aiida.orm import Code, Computer, load_code +from aiida.orm import Computer, InstalledCode, PortableCode, load_code from aiida.plugins import DataFactory @pytest.fixture def code(aiida_localhost): """Return a ``Code`` instance.""" - code = Code( - input_plugin_name='core.arithmetic.add', - remote_computer_exec=[aiida_localhost, '/remote/abs/path'], + code = InstalledCode( + default_calc_job_plugin='core.arithmetic.add', + computer=aiida_localhost, + filepath_executable='/remote/abs/path', ) code.label = 'code' code.description = 'desc' - code.set_prepend_text('text to prepend') - code.set_append_text('text to append') + code.prepend_text = 'text to prepend' + code.append_text = 'text to append' code.store() return code @@ -55,9 +58,10 @@ def test_code_list_no_codes_error_message(run_cli_command): @pytest.mark.usefixtures('aiida_profile_clean') def test_code_list(run_cli_command, code): """Test ``verdi code list``.""" - code2 = Code( - input_plugin_name='core.templatereplacer', - remote_computer_exec=[code.computer, '/remote/abs/path'], + code2 = InstalledCode( + default_calc_job_plugin='core.templatereplacer', + computer=code.computer, + filepath_executable='/remote/abs/path', ) code2.label = 'code2' code2.store() @@ -73,7 +77,7 @@ def test_code_list(run_cli_command, code): @pytest.mark.usefixtures('aiida_profile_clean') def test_code_list_hide(run_cli_command, code): """Test that hidden codes are shown (or not) properly.""" - code.hide() + code.is_hidden = True options = ['-A'] result = run_cli_command(cmd_code.code_list, options) assert code.full_label not in result.output @@ -87,14 +91,14 @@ def test_code_list_hide(run_cli_command, code): def test_hide_one(run_cli_command, code): """Test ``verdi code hide``.""" run_cli_command(cmd_code.hide, [str(code.pk)]) - assert code.hidden + assert code.is_hidden @pytest.mark.usefixtures('aiida_profile_clean') def test_reveal_one(run_cli_command, code): """Test ``verdi code reveal``.""" run_cli_command(cmd_code.reveal, [str(code.pk)]) - assert not code.hidden + assert not code.is_hidden @pytest.mark.usefixtures('aiida_profile_clean') @@ -104,13 +108,6 @@ def test_relabel_code(run_cli_command, code): assert load_code(code.pk).label == 'new_code' -@pytest.mark.usefixtures('aiida_profile_clean') -def test_relabel_code_full(run_cli_command, code): - """Test ``verdi code relabel`` passing the full code label.""" - run_cli_command(cmd_code.relabel, [str(code.pk), f'new_code@{code.computer.label}']) - assert load_code(code.pk).label == 'new_code' - - @pytest.mark.usefixtures('aiida_profile_clean') def test_relabel_code_full_bad(run_cli_command, code): """Test ``verdi code relabel`` with an incorrect full code label.""" @@ -141,9 +138,9 @@ def test_code_duplicate_non_interactive(run_cli_command, code, non_interactive_e duplicate = load_code(label) assert code.description == duplicate.description - assert code.get_prepend_text() == duplicate.get_prepend_text() - assert code.get_append_text() == duplicate.get_append_text() - assert code.get_input_plugin_name() == duplicate.get_input_plugin_name() + assert code.prepend_text == duplicate.prepend_text + assert code.append_text == duplicate.append_text + assert code.default_calc_job_plugin == duplicate.default_calc_job_plugin @pytest.mark.usefixtures('aiida_profile_clean') @@ -161,7 +158,7 @@ def test_noninteractive_remote(run_cli_command, aiida_localhost, non_interactive '--remote-abs-path=/remote/abs/path', ] run_cli_command(cmd_code.setup_code, options) - assert isinstance(load_code(label), Code) + assert isinstance(load_code(label), InstalledCode) @pytest.mark.usefixtures('aiida_profile_clean') @@ -174,7 +171,7 @@ def test_noninteractive_upload(run_cli_command, non_interactive_editor): '--store-in-db', f'--code-folder={os.path.dirname(__file__)}', f'--code-rel-path={os.path.basename(__file__)}' ] run_cli_command(cmd_code.setup_code, options) - assert isinstance(load_code(label), Code) + assert isinstance(load_code(label), PortableCode) @pytest.mark.usefixtures('aiida_profile_clean') @@ -184,7 +181,7 @@ def test_interactive_remote(run_cli_command, aiida_localhost, non_interactive_ed label = 'interactive_remote' user_input = '\n'.join(['yes', aiida_localhost.label, label, 'desc', 'core.arithmetic.add', '/remote/abs/path']) run_cli_command(cmd_code.setup_code, user_input=user_input) - assert isinstance(load_code(label), Code) + assert isinstance(load_code(label), InstalledCode) @pytest.mark.usefixtures('aiida_profile_clean') @@ -196,7 +193,7 @@ def test_interactive_upload(run_cli_command, non_interactive_editor): basename = os.path.basename(__file__) user_input = '\n'.join(['no', label, 'description', 'core.arithmetic.add', dirname, basename]) run_cli_command(cmd_code.setup_code, user_input=user_input) - assert isinstance(load_code(label), Code) + assert isinstance(load_code(label), PortableCode) @pytest.mark.usefixtures('aiida_profile_clean') @@ -207,7 +204,7 @@ def test_mixed(run_cli_command, aiida_localhost, non_interactive_editor): options = ['--description=description', '--on-computer', '--remote-abs-path=/remote/abs/path'] user_input = '\n'.join([aiida_localhost.label, label, 'core.arithmetic.add']) run_cli_command(cmd_code.setup_code, options, user_input=user_input) - assert isinstance(load_code(label), Code) + assert isinstance(load_code(label), InstalledCode) @pytest.mark.usefixtures('aiida_profile_clean') @@ -221,8 +218,8 @@ def test_code_duplicate_interactive(run_cli_command, aiida_local_code_factory, n duplicate = load_code(label) assert code.description == duplicate.description - assert code.get_prepend_text() == duplicate.get_prepend_text() - assert code.get_append_text() == duplicate.get_append_text() + assert code.prepend_text == duplicate.prepend_text + assert code.append_text == duplicate.append_text @pytest.mark.usefixtures('aiida_profile_clean') @@ -259,7 +256,7 @@ def test_from_config_local_file(non_interactive_editor, run_cli_command, aiida_l handle.write(config_file_template.format(label=label, computer=aiida_localhost.label)) handle.flush() run_cli_command(cmd_code.setup_code, ['--non-interactive', '--config', os.path.realpath(handle.name)]) - assert isinstance(load_code(label), Code) + assert isinstance(load_code(label), InstalledCode) @pytest.mark.usefixtures('aiida_profile_clean') @@ -285,7 +282,7 @@ def test_from_config_url(non_interactive_editor, run_cli_command, aiida_localhos label = 'noninteractive_config_url' fake_url = 'https://my.url.com' run_cli_command(cmd_code.setup_code, ['--non-interactive', '--config', fake_url]) - assert isinstance(load_code(label), Code) + assert isinstance(load_code(label), InstalledCode) @pytest.mark.usefixtures('aiida_profile_clean') @@ -296,12 +293,12 @@ def test_code_setup_remote_duplicate_full_label_interactive( """Test ``verdi code setup`` for a remote code in interactive mode specifying an existing full label.""" label = 'some-label' aiida_local_code_factory('core.arithmetic.add', '/bin/cat', computer=aiida_localhost, label=label) - assert isinstance(load_code(label), Code) + assert isinstance(load_code(label), InstalledCode) label_unique = 'label-unique' user_input = '\n'.join(['yes', aiida_localhost.label, label, label_unique, 'd', 'core.arithmetic.add', '/bin/bash']) run_cli_command(cmd_code.setup_code, user_input=user_input) - assert isinstance(load_code(label_unique), Code) + assert isinstance(load_code(label_unique), InstalledCode) @pytest.mark.usefixtures('aiida_profile_clean') @@ -312,7 +309,7 @@ def test_code_setup_remote_duplicate_full_label_non_interactive( """Test ``verdi code setup`` for a remote code in non-interactive mode specifying an existing full label.""" label = f'some-label-{label_first}' aiida_local_code_factory('core.arithmetic.add', '/bin/cat', computer=aiida_localhost, label=label) - assert isinstance(load_code(label), Code) + assert isinstance(load_code(label), InstalledCode) options = ['-n', '-D', 'd', '-P', 'core.arithmetic.add', '--on-computer', '--remote-abs-path=/remote/abs/path'] @@ -335,15 +332,15 @@ def test_code_setup_local_duplicate_full_label_interactive( filepath.write_text('fake bash') label = 'some-label' - code = Code(local_executable='bash', files=[filepath]) + code = PortableCode(filepath_executable='bash', filepath_files=tmp_path) code.label = label code.store() - assert isinstance(load_code(label), Code) + assert isinstance(load_code(label), PortableCode) label_unique = 'label-unique' user_input = '\n'.join(['no', label, label_unique, 'd', 'core.arithmetic.add', str(tmp_path), filepath.name]) run_cli_command(cmd_code.setup_code, user_input=user_input) - assert isinstance(load_code(label_unique), Code) + assert isinstance(load_code(label_unique), PortableCode) @pytest.mark.usefixtures('aiida_profile_clean') @@ -352,10 +349,11 @@ def test_code_setup_local_duplicate_full_label_non_interactive( ): """Test ``verdi code setup`` for a local code in non-interactive mode specifying an existing full label.""" label = 'some-label' - code = Code(local_executable='bash', files=['/bin/bash']) + code = PortableCode(filepath_executable='bash', filepath_files=pathlib.Path('/bin/bash')) code.label = label + code.base.repository.put_object_from_filelike(io.BytesIO(b''), 'bash') code.store() - assert isinstance(load_code(label), Code) + assert isinstance(load_code(label), PortableCode) options = [ '-n', '-D', 'd', '-P', 'core.arithmetic.add', '--store-in-db', '--code-folder=/bin', '--code-rel-path=bash', @@ -395,7 +393,7 @@ def test_code_test(run_cli_command): computer = Computer( label='test-code-computer', transport_type='core.local', hostname='localhost', scheduler_type='core.slurm' ).store() - code = Code(remote_computer_exec=[computer, '/bin/invalid']).store() + code = InstalledCode(computer=computer, filepath_executable='/bin/invalid').store() result = run_cli_command(cmd_code.code_test, [str(code.pk)], raises=True) assert 'Could not connect to the configured computer' in result.output @@ -403,9 +401,9 @@ def test_code_test(run_cli_command): computer.configure() result = run_cli_command(cmd_code.code_test, [str(code.pk)], raises=True) - assert 'the provided remote absolute path `/bin/invalid` does not exist' in result.output + assert 'The provided remote absolute path `/bin/invalid` does not exist' in result.output - code = Code(remote_computer_exec=[computer, '/bin/bash']).store() + code = InstalledCode(computer=computer, filepath_executable='/bin/bash').store() result = run_cli_command(cmd_code.code_test, [str(code.pk)]) assert 'all tests succeeded.' in result.output diff --git a/tests/cmdline/commands/test_run.py b/tests/cmdline/commands/test_run.py index 6fd599cca1..9ee4b0fc92 100644 --- a/tests/cmdline/commands/test_run.py +++ b/tests/cmdline/commands/test_run.py @@ -166,12 +166,12 @@ def test_no_autogroup(self): @pytest.mark.requires_rmq def test_autogroup_filter_class(self): # pylint: disable=too-many-locals """Check if the autogroup is properly generated but filtered classes are skipped.""" - from aiida.orm import AutoGroup, Code, Node, QueryBuilder, load_node + from aiida.orm import AutoGroup, Node, QueryBuilder, load_node script_content = textwrap.dedent( """\ import sys - from aiida.orm import Computer, Int, ArrayData, KpointsData, CalculationNode, WorkflowNode + from aiida.orm import Computer, Int, ArrayData, KpointsData, CalculationNode, WorkflowNode, InstalledCode from aiida.plugins import CalculationFactory from aiida.engine import run_get_node ArithmeticAdd = CalculationFactory('core.arithmetic.add') @@ -186,9 +186,10 @@ def test_autogroup_filter_class(self): # pylint: disable=too-many-locals ).store() computer.configure() - code = Code( - input_plugin_name='core.arithmetic.add', - remote_computer_exec=[computer, '/bin/true']).store() + code = InstalledCode( + default_calc_job_plugin='core.arithmetic.add', + computer=computer, + filepath_executable='/bin/true').store() inputs = { 'x': Int(1), 'y': Int(2), @@ -218,7 +219,6 @@ def test_autogroup_filter_class(self): # pylint: disable=too-many-locals """ ) - Code() for idx, ( flags, kptdata_in_autogroup, diff --git a/tests/cmdline/params/types/test_code.py b/tests/cmdline/params/types/test_code.py index df55aa888d..74e4497cda 100644 --- a/tests/cmdline/params/types/test_code.py +++ b/tests/cmdline/params/types/test_code.py @@ -13,7 +13,7 @@ import pytest from aiida.cmdline.params.types import CodeParamType -from aiida.orm import Code +from aiida.orm import InstalledCode from aiida.orm.utils.loaders import OrmEntityLoader @@ -31,11 +31,13 @@ def setup_codes(aiida_profile_clean, aiida_localhost): the ID and UUID, respectively, of the first one. This allows us to test the rules implemented to solve ambiguities that arise when determing the identifier type. """ - entity_01 = Code(remote_computer_exec=(aiida_localhost, '/bin/true')).store() - entity_02 = Code(remote_computer_exec=(aiida_localhost, '/bin/true'), - input_plugin_name='core.arithmetic.add').store() - entity_03 = Code(remote_computer_exec=(aiida_localhost, '/bin/true'), - input_plugin_name='core.templatereplacer').store() + entity_01 = InstalledCode(computer=aiida_localhost, filepath_executable='/bin/true').store() + entity_02 = InstalledCode( + computer=aiida_localhost, filepath_executable='/bin/true', default_calc_job_plugin='core.arithmetic.add' + ).store() + entity_03 = InstalledCode( + computer=aiida_localhost, filepath_executable='/bin/true', default_calc_job_plugin='core.templatereplacer' + ).store() entity_01.label = 'computer_01' entity_02.label = str(entity_01.pk) @@ -124,7 +126,7 @@ def test_entry_point_validation(setup_codes): def test_shell_complete(setup_codes, parameter_type, aiida_localhost): """Test the `shell_complete` method that provides auto-complete functionality.""" entity_01, entity_02, entity_03 = setup_codes - entity_04 = Code(label='xavier', remote_computer_exec=(aiida_localhost, '/bin/true')).store() + entity_04 = InstalledCode(label='xavier', computer=aiida_localhost, filepath_executable='/bin/true').store() options = [item.value for item in parameter_type.shell_complete(None, None, '')] assert sorted(options) == sorted([entity_01.label, entity_02.label, entity_03.label, entity_04.label]) diff --git a/tests/conftest.py b/tests/conftest.py index f9550f70f6..e9de501f7a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -373,6 +373,21 @@ def override_logging(isolated_config): configure_logging(with_orm=True) +@pytest.fixture +def suppress_internal_deprecations(): + """Suppress all internal deprecations. + + Warnings emmitted of type :class:`aiida.common.warnings.AiidaDeprecationWarning` for the duration of the test. + """ + import warnings + + from aiida.common.warnings import AiidaDeprecationWarning + + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=AiidaDeprecationWarning) + yield + + @pytest.fixture def with_daemon(): """Starts the daemon process and then makes sure to kill it once the test is done.""" diff --git a/tests/engine/processes/calcjobs/test_calc_job.py b/tests/engine/processes/calcjobs/test_calc_job.py index 1003b6064e..85e96e83ee 100644 --- a/tests/engine/processes/calcjobs/test_calc_job.py +++ b/tests/engine/processes/calcjobs/test_calc_job.py @@ -201,8 +201,8 @@ def test_computer_double_quotes(aiida_local_code_factory, file_regression, compu @pytest.mark.parametrize('code_use_double_quotes', [True, False]) def test_code_double_quotes(aiida_localhost, file_regression, code_use_double_quotes): """test that bash script quote escape behaviour can be controlled""" - code = orm.Code(remote_computer_exec=(aiida_localhost, '/bin/bash')) - code.set_use_double_quotes(code_use_double_quotes) + code = orm.InstalledCode(computer=aiida_localhost, filepath_executable='/bin/bash') + code.use_double_quotes = code_use_double_quotes inputs = { 'code': code.store(), 'metadata': { @@ -261,13 +261,15 @@ class TestCalcJob: """Test for the `CalcJob` process sub class.""" @pytest.fixture(autouse=True) - def init_profile(self, aiida_profile_clean, aiida_localhost): # pylint: disable=unused-argument + def init_profile(self, aiida_profile_clean, aiida_localhost, tmp_path): # pylint: disable=unused-argument """Initialize the profile.""" # pylint: disable=attribute-defined-outside-init + (tmp_path / 'bash').write_bytes(b'bash implementation') + assert Process.current() is None self.computer = aiida_localhost - self.remote_code = orm.Code(remote_computer_exec=(self.computer, '/bin/bash')).store() - self.local_code = orm.Code(local_executable='bash', files=['/bin/bash']).store() + self.remote_code = orm.InstalledCode(computer=self.computer, filepath_executable='/bin/bash').store() + self.local_code = orm.PortableCode(filepath_executable='bash', filepath_files=tmp_path).store() self.inputs = {'x': orm.Int(1), 'y': orm.Int(2), 'metadata': {'options': {}}} yield assert Process.current() is None @@ -434,7 +436,7 @@ def test_par_env_resources_computer(self): computer = orm.Computer('sge_computer', 'localhost', 'desc', 'core.local', 'core.sge').store() computer.set_default_mpiprocs_per_machine(1) - inputs['code'] = orm.Code(remote_computer_exec=(computer, '/bin/bash')).store() + inputs['code'] = orm.InstalledCode(computer=computer, filepath_executable='/bin/bash').store() inputs['metadata']['options']['resources'] = {'parallel_env': 'environment', 'tot_num_mpiprocs': 10} # Just checking that instantiating does not raise, meaning the inputs were valid @@ -493,8 +495,9 @@ def test_rerunnable(self): @pytest.mark.usefixtures('chdir_tmp_path') def test_provenance_exclude_list(self): """Test the functionality of the `CalcInfo.provenance_exclude_list` attribute.""" - code = orm.Code(input_plugin_name='core.arithmetic.add', remote_computer_exec=[self.computer, - '/bin/true']).store() + code = orm.InstalledCode( + default_calc_job_plugin='core.arithmetic.add', computer=self.computer, filepath_executable='/bin/true' + ).store() with tempfile.NamedTemporaryFile('w+') as handle: handle.write('dummy_content') diff --git a/tests/engine/processes/test_builder.py b/tests/engine/processes/test_builder.py index 21d9aa4112..817e255dba 100644 --- a/tests/engine/processes/test_builder.py +++ b/tests/engine/processes/test_builder.py @@ -122,7 +122,7 @@ def test_builder_inputs(): # When no inputs are specified specifically, `prune=True` should get rid of completely empty namespaces assert builder._inputs(prune=False) == {'namespace': {'nested': {}}, 'metadata': {}} - assert builder._inputs(prune=True) == {} + assert not builder._inputs(prune=True) # With a specific input in `namespace` the case of `prune=True` should now only remove `metadata` integer = orm.Int(DEFAULT_INT) @@ -313,10 +313,12 @@ def test_calc_job_node_get_builder_restart(aiida_localhost): def test_code_get_builder(aiida_localhost): """Test that the `Code.get_builder` method returns a builder where the code is already set.""" - code = orm.Code() - code.set_remote_computer_exec((aiida_localhost, '/bin/true')) - code.label = 'test_code' - code.set_input_plugin_name('core.templatereplacer') + code = orm.InstalledCode( + label='test_code', + computer=aiida_localhost, + filepath_executable='/bin/true', + default_calc_job_plugin='core.templatereplacer' + ) code.store() # Check that I can get a builder diff --git a/tests/engine/test_launch.py b/tests/engine/test_launch.py index c187ac31c8..7f7fb89af8 100644 --- a/tests/engine/test_launch.py +++ b/tests/engine/test_launch.py @@ -173,8 +173,9 @@ def test_launchers_dry_run(self): ArithmeticAddCalculation = CalculationFactory('core.arithmetic.add') # pylint: disable=invalid-name - code = orm.Code(input_plugin_name='core.arithmetic.add', remote_computer_exec=[self.computer, - '/bin/true']).store() + code = orm.InstalledCode( + default_calc_job_plugin='core.arithmetic.add', computer=self.computer, filepath_executable='/bin/true' + ).store() inputs = { 'code': code, @@ -214,8 +215,9 @@ def test_launchers_dry_run_no_provenance(self): ArithmeticAddCalculation = CalculationFactory('core.arithmetic.add') # pylint: disable=invalid-name - code = orm.Code(input_plugin_name='core.arithmetic.add', remote_computer_exec=[self.computer, - '/bin/true']).store() + code = orm.InstalledCode( + default_calc_job_plugin='core.arithmetic.add', computer=self.computer, filepath_executable='/bin/true' + ).store() inputs = { 'code': code, @@ -262,8 +264,9 @@ def test_calcjob_dry_run_no_provenance(self): """ import tempfile - code = orm.Code(input_plugin_name='core.arithmetic.add', remote_computer_exec=[self.computer, - '/bin/true']).store() + code = orm.InstalledCode( + default_calc_job_plugin='core.arithmetic.add', computer=self.computer, filepath_executable='/bin/true' + ).store() with tempfile.NamedTemporaryFile('w+') as handle: handle.write('dummy_content') diff --git a/tests/engine/test_process.py b/tests/engine/test_process.py index 01de7bbd75..708456976f 100644 --- a/tests/engine/test_process.py +++ b/tests/engine/test_process.py @@ -238,12 +238,9 @@ def test_valid_cache_hook(self): def test_process_type_with_entry_point(self): """For a process with a registered entry point, the process_type will be its formatted entry point string.""" - from aiida.orm import Code - - code = Code() - code.set_remote_computer_exec((self.computer, '/bin/true')) - code.store() + from aiida.orm import InstalledCode + code = InstalledCode(computer=self.computer, filepath_executable='/bin/true').store() parameters = orm.Dict(dict={}) template = orm.Dict(dict={}) options = { diff --git a/tests/orm/data/test_code.py b/tests/orm/data/test_code.py index 2b06c58698..a1907d5ff2 100644 --- a/tests/orm/data/test_code.py +++ b/tests/orm/data/test_code.py @@ -15,13 +15,13 @@ from aiida.orm import Code, Computer -@pytest.mark.usefixtures('aiida_profile_clean') +@pytest.mark.usefixtures('aiida_profile_clean', 'suppress_internal_deprecations') def test_validate_remote_exec_path(): """Test ``Code.validate_remote_exec_path``.""" computer = Computer( label='test-code-computer', transport_type='core.local', hostname='localhost', scheduler_type='core.slurm' ).store() - code = Code(remote_computer_exec=[computer, '/bin/invalid']) + code = Code(remote_computer_exec=(computer, '/bin/invalid')) with pytest.raises(ValidationError, match=r'Could not connect to the configured computer.*'): code.validate_remote_exec_path() @@ -31,5 +31,5 @@ def test_validate_remote_exec_path(): with pytest.raises(ValidationError, match=r'the provided remote absolute path `.*` does not exist.*'): code.validate_remote_exec_path() - code = Code(remote_computer_exec=[computer, '/bin/bash']) + code = Code(remote_computer_exec=(computer, '/bin/bash')) code.validate_remote_exec_path() diff --git a/tests/orm/utils/test_loaders.py b/tests/orm/utils/test_loaders.py index 3171926190..9a96ed5a01 100644 --- a/tests/orm/utils/test_loaders.py +++ b/tests/orm/utils/test_loaders.py @@ -52,13 +52,10 @@ def test_load_entity(self): def test_load_code(self): """Test the functionality of load_code.""" - from aiida.orm import Code + from aiida.orm import InstalledCode label = 'compy' - code = Code() - code.label = label - code.set_remote_computer_exec((self.computer, '/x.x')) - code.store() + code = InstalledCode(label=label, computer=self.computer, filepath_executable='/x.x').store() # Load through full label loaded_code = load_code(code.full_label) diff --git a/tests/test_generic.py b/tests/test_generic.py index 024a35b39b..fd9776f52f 100644 --- a/tests/test_generic.py +++ b/tests/test_generic.py @@ -14,8 +14,12 @@ from aiida import orm +@pytest.mark.usefixtures('suppress_internal_deprecations') def test_code_local(aiida_profile_clean, aiida_localhost): - """Test local code.""" + """Test local code. + + Remove this test when legacy `Code` is removed in v3.0. + """ import tempfile from aiida.common.exceptions import ValidationError @@ -37,8 +41,12 @@ def test_code_local(aiida_profile_clean, aiida_localhost): assert code.get_execname(), 'stest.sh' +@pytest.mark.usefixtures('suppress_internal_deprecations') def test_code_remote(aiida_profile_clean, aiida_localhost): - """Test remote code.""" + """Test remote code. + + Remove this test when legacy `Code` is removed in v3.0. + """ import tempfile from aiida.common.exceptions import ValidationError diff --git a/tests/test_nodes.py b/tests/test_nodes.py index 5e959f4c63..48f24e9318 100644 --- a/tests/test_nodes.py +++ b/tests/test_nodes.py @@ -1023,6 +1023,7 @@ def test_comments(self): (default_user_email, 'text2'), ] + @pytest.mark.usefixtures('suppress_internal_deprecations') def test_code_loading_from_string(self): """ Checks that the method Code.get_from_string works correctly. @@ -1070,6 +1071,7 @@ def test_code_loading_from_string(self): with pytest.raises(MultipleObjectsError): orm.Code.get_from_string(code3.label) + @pytest.mark.usefixtures('suppress_internal_deprecations') def test_code_loading_using_get(self): """ Checks that the method Code.get(pk) works correctly. @@ -1129,7 +1131,7 @@ def test_code_loading_using_get(self): pk_label_duplicate = code1.pk code4 = orm.Code() code4.set_remote_computer_exec((self.computer, '/bin/true')) - code4.label = pk_label_duplicate + code4.label = str(pk_label_duplicate) code4.store() # Since the label of code4 is identical to the pk of code1, calling @@ -1140,24 +1142,7 @@ def test_code_loading_using_get(self): assert q_code_4.label == code1.label assert q_code_4.get_remote_exec_path() == code1.get_remote_exec_path() - def test_code_description(self): - """ - This test checks that the code description is retrieved correctly - when the code is searched with its id and label. - """ - # Create a code node - code = orm.Code() - code.set_remote_computer_exec((self.computer, '/bin/true')) - code.label = 'test_code_label' - code.description = 'test code description' - code.store() - - q_code1 = orm.Code.get(label=code.label) - assert code.description == str(q_code1.description) - - q_code2 = orm.Code.get(code.pk) - assert code.description == str(q_code2.description) - + @pytest.mark.usefixtures('suppress_internal_deprecations') def test_list_for_plugin(self): """ This test checks the Code.list_for_plugin() diff --git a/tests/tools/archive/migration/test_prov_redesign.py b/tests/tools/archive/migration/test_prov_redesign.py index 3526f25ca1..f026554392 100644 --- a/tests/tools/archive/migration/test_prov_redesign.py +++ b/tests/tools/archive/migration/test_prov_redesign.py @@ -133,14 +133,13 @@ def test_node_process_type(aiida_profile, tmp_path): assert node.process_type == node_process_type +@pytest.mark.usefixtures('suppress_internal_deprecations') def test_code_type_change(aiida_profile_clean, tmp_path, aiida_localhost): """ Code type string changed Change: “code.Bool.” → “data.code.Code.” """ # Create Code instance - code = orm.Code() - code.set_remote_computer_exec((aiida_localhost, '/bin/true')) - code.store() + code = orm.Code((aiida_localhost, '/bin/true')).store() # Save uuid and type code_uuid = str(code.uuid) diff --git a/tests/tools/archive/orm/test_codes.py b/tests/tools/archive/orm/test_codes.py index a216353348..c93e6d33b0 100644 --- a/tests/tools/archive/orm/test_codes.py +++ b/tests/tools/archive/orm/test_codes.py @@ -21,11 +21,7 @@ def test_that_solo_code_is_exported_correctly(tmp_path, aiida_profile_clean, aii """ code_label = 'test_code1' - code = orm.Code() - code.set_remote_computer_exec((aiida_localhost, '/bin/true')) - code.label = code_label - code.store() - + code = orm.InstalledCode(label=code_label, computer=aiida_localhost, filepath_executable='/bin/true').store() code_uuid = code.uuid export_file = tmp_path / 'export.aiida' @@ -46,11 +42,7 @@ def test_input_code(tmp_path, aiida_profile_clean, aiida_localhost): """ code_label = 'test_code1' - code = orm.Code() - code.set_remote_computer_exec((aiida_localhost, '/bin/true')) - code.label = code_label - code.store() - + code = orm.InstalledCode(label=code_label, computer=aiida_localhost, filepath_executable='/bin/true').store() code_uuid = code.uuid calc = orm.CalcJobNode() @@ -90,11 +82,7 @@ def test_solo_code(tmp_path, aiida_profile_clean, aiida_localhost): """ code_label = 'test_code1' - code = orm.Code() - code.set_remote_computer_exec((aiida_localhost, '/bin/true')) - code.label = code_label - code.store() - + code = orm.InstalledCode(label=code_label, computer=aiida_localhost, filepath_executable='/bin/true').store() code_uuid = code.uuid export_file = tmp_path / 'export.aiida' From abed94f922c2cc6630fde0f550731412d728a56c Mon Sep 17 00:00:00 2001 From: Sebastiaan Huber Date: Mon, 16 May 2022 20:17:50 +0200 Subject: [PATCH 6/7] Deprecate: `verdi code setup` and `CodeBuilder` The CLI command is replaced with `verdi code create`. The `CodeBuilder` is no longer necessary because code instances can now easily be created through the constructor. --- aiida/cmdline/commands/cmd_code.py | 3 ++- aiida/orm/utils/builders/code.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/aiida/cmdline/commands/cmd_code.py b/aiida/cmdline/commands/cmd_code.py index a7f1d0a5b1..2050a1564a 100644 --- a/aiida/cmdline/commands/cmd_code.py +++ b/aiida/cmdline/commands/cmd_code.py @@ -18,7 +18,7 @@ from aiida.cmdline.params import arguments, options from aiida.cmdline.params.options.commands import code as options_code from aiida.cmdline.utils import echo -from aiida.cmdline.utils.decorators import with_dbenv +from aiida.cmdline.utils.decorators import deprecated_command, with_dbenv from aiida.common import exceptions @@ -85,6 +85,7 @@ def set_code_builder(ctx, param, value): @options.CONFIG_FILE() @click.pass_context @with_dbenv() +@deprecated_command('This command will be removed soon, use `verdi code create` instead.') def setup_code(ctx, non_interactive, **kwargs): """Setup a new code.""" from aiida.orm.utils.builders.code import CodeBuilder diff --git a/aiida/orm/utils/builders/code.py b/aiida/orm/utils/builders/code.py index 6028748fcc..bd110ffb5b 100644 --- a/aiida/orm/utils/builders/code.py +++ b/aiida/orm/utils/builders/code.py @@ -13,8 +13,11 @@ from aiida.cmdline.utils.decorators import with_dbenv from aiida.common.utils import ErrorAccumulator +from aiida.common.warnings import warn_deprecation from aiida.orm import InstalledCode, PortableCode +warn_deprecation('This module is deprecated. To create a new code instance, simply use the constructor.', version=3) + class CodeBuilder: """Build a code with validation of attribute combinations""" From 2d3cf681c2f71e59b3894449210461e94c3e199b Mon Sep 17 00:00:00 2001 From: Sebastiaan Huber Date: Tue, 17 May 2022 12:40:01 +0200 Subject: [PATCH 7/7] Docs: add documentation on new `InstalledCode` and `PortableCode` It also provides some historical context as to why it replaces the old `Code` and when that will be removed. --- docs/source/topics/data_types.rst | 151 +++++++++++++++++++++++------- 1 file changed, 119 insertions(+), 32 deletions(-) diff --git a/docs/source/topics/data_types.rst b/docs/source/topics/data_types.rst index 2beb07147d..860db0dcad 100644 --- a/docs/source/topics/data_types.rst +++ b/docs/source/topics/data_types.rst @@ -40,35 +40,43 @@ Below is a list of the core data types already provided with AiiDA, along with t .. table:: :widths: 20 20 45 45 - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | **Class** | **Entry point** | **Stored in database** | **Stored in repository** | - +===========================================================+========================+===================================================+===================================+ - | :ref:`Int ` | ``core.int`` | The integer value | \\- | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`Float ` | ``core.float`` | The float value | \\- | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`Str ` | ``core.str`` | The string | \\- | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`Bool ` | ``core.bool`` | The boolean value | \\- | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`List ` | ``core.list`` | The complete list | \\- | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`Dict ` | ``core.dict`` | The complete dictionary | \\- | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`EnumData ` | ``core.enum`` | The value, name and the class identifier | \\- | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`JsonableData ` | ``core.jsonable`` | The JSON data and the class identifier | \\- | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`ArrayData ` | ``core.array`` | The array names and corresponding shapes | The array data in ``.npy`` format | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`XyData ` | ``core.array.xy`` | The array names and corresponding shapes | The array data in ``.npy`` format | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`SinglefileData ` | ``core.singlefile`` | The filename | The file | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`FolderData ` | ``core.folder`` | \\- | All files and folders | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`RemoteData ` | ``core.remote`` | The computer and the absolute path to the folder | All files and folders | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | **Class** | **Entry point** | **Stored in database** | **Stored in repository** | + +==============================================================+========================+===================================================+===================================+ + | :ref:`Int ` | ``core.int`` | The integer value | ``-`` | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`Float ` | ``core.float`` | The float value | ``-`` | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`Str ` | ``core.str`` | The string | ``-`` | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`Bool ` | ``core.bool`` | The boolean value | ``-`` | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`List ` | ``core.list`` | The complete list | ``-`` | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`Dict ` | ``core.dict`` | The complete dictionary | ``-`` | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`EnumData ` | ``core.enum`` | The value, name and the class identifier | ``-`` | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`JsonableData ` | ``core.jsonable`` | The JSON data and the class identifier | ``-`` | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`ArrayData ` | ``core.array`` | The array names and corresponding shapes | The array data in ``.npy`` format | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`XyData ` | ``core.array.xy`` | The array names and corresponding shapes | The array data in ``.npy`` format | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`SinglefileData ` | ``core.singlefile`` | The filename | The file | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`FolderData ` | ``core.folder`` | ``-`` | All files and folders | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`RemoteData ` | ``core.remote`` | The computer and the absolute path to the folder | All files and folders | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`AbstractCode ` | ``-`` | Default plugin, append/prepend text | ``-`` | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`Code ` | ``core.code`` | The computer and the executable path | All files and folders | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`InstalledCode ` | ``core.code.installed``| The computer and the executable path | ``-`` | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`PortableCode ` | ``core.code.portable`` | The relative path of the executable | All files and folders of the code | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ .. _topics:data_types:core:base: @@ -400,11 +408,90 @@ To see the contents of a subdirectory, pass the relative path to the :py:meth:`~ Using the :py:meth:`~aiida.orm.RemoteData.listdir()` method, or any method that retrieves information from the remote computer, opens a connection to the remote computer using its transport type. Their use is strongly discouraged when writing scripts and/or workflows. -.. todo:: - .. _topics:data_types:core:code: +.. _topics:data_types:core:code: + +AbstractCode +------------ + +.. versionadded:: 2.1 + +The :class:`aiida.orm.nodes.data.code.abstract.AbstractCode` class provides the abstract class for objects that represent a "code" that can be executed through a :class:`aiida.engine.processes.calcjobs.calcjob.CalcJob` plugin. +There are currently three implementations of this abstract class: + + * :class:`~aiida.orm.nodes.data.code.legacy.Code` (see :ref:`Code `) + * :class:`~aiida.orm.nodes.data.code.installed.InstalledCode` (see :ref:`InstalledCode `) + * :class:`~aiida.orm.nodes.data.code.portable.PortableCode` (see :ref:`PortableCode `) + + +.. _topics:data_types:core:code:legacy: + +Code +---- + +.. deprecated:: 2.1 + +Historically, there was only one code implementation, the :class:`~aiida.orm.nodes.data.code.legacy.Code`, which implemented two different types of code: + + * An executable pre-installed on a computer, represented by a :class:`~aiida.orm.computers.Computer`. + * A directory containing all code files including an executable which would be uploaded to + +These two types were referred to as "remote" and "local" codes. +However, this nomenclature would lead to confusion as a "remote" code could also refer to an executable on the localhost, i.e., the machine where AiiDA itself runs. +In addition, having two different concepts implemented by a single class led to a unintuitive interface. +Therefore, the ``Code`` class was deprecated in ``aiida-core==2.1`` and replaced by the :ref:`InstallCode ` and :ref:`InstallCode `, respectively. +The ``Code`` class is now deprecated and will be removed in ``aiida-core==3.0``. + + +.. _topics:data_types:core:code:installed: + +InstalledCode +------------- + +.. versionadded:: 2.1 + +The :class:`~aiida.orm.nodes.data.code.installed.InstalledCode` class is an implementation of the :class:`~aiida.orm.nodes.data.code.abstract.AbstractCode` class that represents an executable code on a remote computer. +This plugin should be used if an executable is pre-installed on a computer. +The ``InstalledCode`` represents the code by storing the absolute filepath of the relevant executable and the computer on which it is installed. +The computer is represented by an instance of :class:`~aiida.orm.computers.Computer`. +Each time a :class:`~aiida.engine.CalcJob` is run using an ``InstalledCode``, it will run its executable on the associated computer. +Example of creating an ``InstalledCode``: + +.. code:: python + + from aiida.orm import InstalledCode + code = InstalledCode( + label='some-label', + computer=load_computer('localhost'), + filepath_executable='/usr/bin/bash' + ) + + +.. _topics:data_types:core:code:portable: + +PortableCode +------------ + +.. versionadded:: 2.1 + +The :class:`~aiida.orm.nodes.data.code.portable.PortableCode` class is an implementation of the :class:`~aiida.orm.nodes.data.code.abstract.AbstractCode` class that represents an executable code stored in AiiDA's storage. +This plugin should be used for executables that are not already installed on the target computer, but instead are available on the machine where AiiDA is running. +The plugin assumes that the code is self-contained by a single directory containing all the necessary files, including a main executable. +When constructing a ``PortableCode``, passing the absolute filepath as ``filepath_files`` will make sure that all the files contained within are uploaded to AiiDA's storage. +The ``filepath_executable`` should indicate the filename of the executable within that directory. +Each time a :class:`~aiida.engine.CalcJob` is run using a ``PortableCode``, the uploaded files will be automatically copied to the working directory on the selected computer and the executable will be run there. +Example of creating an ``PortableCode``: + +.. code:: python + + from pathlib import Path + from aiida.orm import PortableCode + code = PortableCode( + label='some-label', + filepath_files=Path('/some/path/code'), + filepath_executable='executable.exe' + ) - title: Code .. _topics:data_types:materials: