diff --git a/.github/system_tests/pytest/test_memory_leaks.py b/.github/system_tests/pytest/test_memory_leaks.py index 4632d9bc3b..396793346f 100644 --- a/.github/system_tests/pytest/test_memory_leaks.py +++ b/.github/system_tests/pytest/test_memory_leaks.py @@ -50,8 +50,10 @@ def test_leak_ssh_calcjob(): Note: This relies on the 'slurm-ssh' computer being set up. """ - code = orm.Code( - input_plugin_name='core.arithmetic.add', remote_computer_exec=[orm.load_computer('slurm-ssh'), '/bin/bash'] + code = orm.InstalledCode( + default_calc_job_plugin='core.arithmetic.add', + computer=orm.load_computer('slurm-ssh'), + filepath_executable='/bin/bash' ) inputs = {'x': orm.Int(1), 'y': orm.Int(2), 'code': code} run_finished_ok(ArithmeticAddCalculation, **inputs) diff --git a/aiida/cmdline/__init__.py b/aiida/cmdline/__init__.py index 1eed47d50e..f34e1f5f52 100644 --- a/aiida/cmdline/__init__.py +++ b/aiida/cmdline/__init__.py @@ -14,6 +14,7 @@ # yapf: disable # pylint: disable=wildcard-import +from .groups import * from .params import * from .utils import * @@ -24,6 +25,7 @@ 'ComputerParamType', 'ConfigOptionParamType', 'DataParamType', + 'DynamicEntryPointCommandGroup', 'EmailType', 'EntryPointType', 'FileOrUrl', @@ -42,6 +44,7 @@ 'ProfileParamType', 'ShebangParamType', 'UserParamType', + 'VerdiCommandGroup', 'WorkflowParamType', 'dbenv', 'echo_critical', diff --git a/aiida/cmdline/commands/cmd_code.py b/aiida/cmdline/commands/cmd_code.py index 6b5f07c854..2050a1564a 100644 --- a/aiida/cmdline/commands/cmd_code.py +++ b/aiida/cmdline/commands/cmd_code.py @@ -14,10 +14,11 @@ import tabulate from aiida.cmdline.commands.cmd_verdi import verdi +from aiida.cmdline.groups.dynamic import DynamicEntryPointCommandGroup from aiida.cmdline.params import arguments, options from aiida.cmdline.params.options.commands import code as options_code from aiida.cmdline.utils import echo -from aiida.cmdline.utils.decorators import with_dbenv +from aiida.cmdline.utils.decorators import deprecated_command, with_dbenv from aiida.common import exceptions @@ -26,6 +27,11 @@ def verdi_code(): """Setup and manage codes.""" +@verdi_code.group('create', cls=DynamicEntryPointCommandGroup, entry_point_name_filter=r'core\.code\..*') +def code_create(): + """Create a new code.""" + + def get_default(key, ctx): """ Get the default argument using a user instance property @@ -79,6 +85,7 @@ def set_code_builder(ctx, param, value): @options.CONFIG_FILE() @click.pass_context @with_dbenv() +@deprecated_command('This command will be removed soon, use `verdi code create` instead.') def setup_code(ctx, non_interactive, **kwargs): """Setup a new code.""" from aiida.orm.utils.builders.code import CodeBuilder @@ -106,7 +113,6 @@ def setup_code(ctx, non_interactive, **kwargs): except Exception as exception: # pylint: disable=broad-except echo.echo_critical(f'Unable to store the Code: {exception}') - code.reveal() echo.echo_success(f'Code<{code.pk}> {code.full_label} created') @@ -121,9 +127,11 @@ def code_test(code): * Whether the remote executable exists. """ - if not code.is_local(): + from aiida.orm import InstalledCode + + if isinstance(code, InstalledCode): try: - code.validate_remote_exec_path() + code.validate_filepath_executable() except exceptions.ValidationError as exception: echo.echo_critical(f'validation failed: {exception}') @@ -163,10 +171,11 @@ def code_duplicate(ctx, code, non_interactive, **kwargs): kwargs['code_type'] = CodeBuilder.CodeType.STORE_AND_UPLOAD if kwargs.pop('hide_original'): - code.hide() + code.is_hidden = True # Convert entry point to its name - kwargs['input_plugin'] = kwargs['input_plugin'].name + if kwargs['input_plugin']: + kwargs['input_plugin'] = kwargs['input_plugin'].name code_builder = ctx.code_builder for key, value in kwargs.items(): @@ -175,7 +184,7 @@ def code_duplicate(ctx, code, non_interactive, **kwargs): try: new_code.store() - new_code.reveal() + new_code.is_hidden = False except ValidationError as exception: echo.echo_critical(f'Unable to store the Code: {exception}') @@ -188,31 +197,15 @@ def code_duplicate(ctx, code, non_interactive, **kwargs): def show(code): """Display detailed information for a code.""" from aiida.cmdline import is_verbose - from aiida.repository import FileType table = [] table.append(['PK', code.pk]) table.append(['UUID', code.uuid]) table.append(['Label', code.label]) table.append(['Description', code.description]) - table.append(['Default plugin', code.get_input_plugin_name()]) - - if code.is_local(): - table.append(['Type', 'local']) - table.append(['Exec name', code.get_execname()]) - table.append(['List of files/folders:', '']) - for obj in code.list_objects(): - if obj.file_type == FileType.DIRECTORY: - table.append(['directory', obj.name]) - else: - table.append(['file', obj.name]) - else: - table.append(['Type', 'remote']) - table.append(['Remote machine', code.get_remote_computer().label]) - table.append(['Remote absolute path', code.get_remote_exec_path()]) - - table.append(['Prepend text', code.get_prepend_text()]) - table.append(['Append text', code.get_append_text()]) + table.append(['Default plugin', code.default_calc_job_plugin]) + table.append(['Prepend text', code.prepend_text]) + table.append(['Append text', code.append_text]) if is_verbose(): table.append(['Calculations', len(code.base.links.get_outgoing().all())]) @@ -252,7 +245,7 @@ def _dry_run_callback(pks): def hide(codes): """Hide one or more codes from `verdi code list`.""" for code in codes: - code.hide() + code.is_hidden = True echo.echo_success(f'Code<{code.pk}> {code.full_label} hidden') @@ -262,7 +255,7 @@ def hide(codes): def reveal(codes): """Reveal one or more hidden codes in `verdi code list`.""" for code in codes: - code.reveal() + code.is_hidden = False echo.echo_success(f'Code<{code.pk}> {code.full_label} revealed') @@ -275,7 +268,7 @@ def relabel(code, label): old_label = code.full_label try: - code.relabel(label) + code.label = label except (TypeError, ValueError) as exception: echo.echo_critical(f'invalid code label: {exception}') else: diff --git a/aiida/cmdline/commands/cmd_verdi.py b/aiida/cmdline/commands/cmd_verdi.py index 959c9f4ec9..bcbdbb11ee 100644 --- a/aiida/cmdline/commands/cmd_verdi.py +++ b/aiida/cmdline/commands/cmd_verdi.py @@ -8,99 +8,12 @@ # For further information please visit http://www.aiida.net # ########################################################################### """The main `verdi` click group.""" -import difflib - import click from aiida import __version__ -from aiida.cmdline.params import options, types - -GIU = ( - 'ABzY8%U8Kw0{@klyK?I~3`Ki?#qHQ&IIM|J;6yB`9_+{&w)p(JK}vokj-11jhve8xcx?dZ>+9nwrEF!x*S>9A+EWYrR?6GA-u?jFa+et65GF@1+D{%' - '8{C~xjt%>uVM4RTSS?j2M)XH%T#>M{K$lE2XGD`RS0T67213wbAs!SZmn+;(-m!>f(T@e%@oxd`yRBp9nu+9N`4xv8AS@O$CaQ;7FXzM=ug^$?3ta2551EDL`wK4|Cm' - '%RnJdS#0UFwVweDkcfdNjtUv1N^iSQui#TL(q!FmIeKb!yW4|L`@!@-4x6' - 'B6I^ptRdH+4o0ODM;1_f^}4@LMe@#_YHz0wQdq@d)@n)uYNtAb2OLo&fpBkct5{~3kbRag^_5QG%qrTksHMXAYAQoz1#2wtHCy0}h?CJtzv&@Q?^9r' - 'd&02;isB7NJMMr7F@>$!ELj(sbwzIR4)rnch=oVZrG;8)%R6}FUk*fv2O&!#ZA)$HloK9!es&4Eb+h=OIyWFha(8PPy9u?NqfkuPYg;GO1RVzBLX)7' - 'ORMM>1hEM`-96mGjJ+A!e-_}4X{M|4CkKE~uF4j+LW#6IsFa*_da_mLqzr)E<`%ikthkMO2>65cNMtpDE*VejqZV^MyewPJJAS*VM6jY;QY' - '#g7gOKgPbFg{@;YDL6Gbxxr|2T&BQunB?PBetq?X>jW1hFF7&>EaYkKYqIa_ld(Z@AJT' - '+lJ(Pd;+?<&&M>A0agti19^z3n4Z6_WG}c~_+XHyJI_iau7+V$#YA$pJ~H)yHEVy1D?5^Sw`tb@{nnNNo=eSMZLf0>m^A@7f{y$nb_HJWgLRtZ?x2?*>SwM?JoQ>p|-1ZRU0#+{^UhK22+~o' - 'R9k7rh(GH9y|jm){jY9_xAI4N_EfU#4' - 'taTUXFY4a4l$v=N-+f+w&wuH;Z(6p6#=n8XwlZ;*L&-rcL~T_vEm@#-Xi8&g06!MO+R( list[str]: + """Return the sorted list of subcommands for this group. + + :param ctx: The :class:`click.Context`. + """ + commands = super().list_commands(ctx) + commands.extend([ + entry_point for entry_point in get_entry_point_names(self.entry_point_group) + if re.match(self.entry_point_name_filter, entry_point) + ]) + return sorted(commands) + + def get_command(self, ctx, cmd_name): + """Return the command with the given name. + + :param ctx: The :class:`click.Context`. + :param cmd_name: The name of the command. + :returns: The :class:`click.Command`. + """ + try: + command = self.create_command(cmd_name) + except exceptions.EntryPointError: + command = super().get_command(ctx, cmd_name) + return command + + def create_command(self, entry_point): + """Create a subcommand to create an instance of a particular code subclass""" + cls = self.factory(entry_point) + + def command(non_interactive, **kwargs): # pylint: disable=unused-argument + """Create a new instance.""" + try: + instance = cls(**kwargs) + except Exception as exception: # pylint: disable=broad-except + echo.echo_critical(f'Failed to instantiate `{cls}`: {exception}') + + try: + instance.store() + except Exception as exception: # pylint: disable=broad-except + echo.echo_critical(f'Failed to store instance of `{cls}`: {exception}') + + echo.echo_success(f'Created {cls.__name__}<{instance.pk}>') + + command.__doc__ = cls.__doc__ + + return click.command(entry_point)(self.create_options(entry_point)(command)) + + def create_options(self, entry_point): + """Create the option decorators for the command function for the given entry point. + + :param entry_point: The entry point. + """ + + def apply_options(func): + """Decorate the command function with the appropriate options for the given entry point.""" + func = options.NON_INTERACTIVE()(func) + func = options.CONFIG_FILE()(func) + + options_list = self.list_options(entry_point) + options_list.reverse() + + for option in options_list: + func = option(func) + + return func + + return apply_options + + def list_options(self, entry_point): + """Return the list of options that should be applied to the command for the given entry point. + + :param entry_point: The entry point. + """ + return [self.create_option(*item) for item in self.factory(entry_point).get_cli_options().items()] + + @staticmethod + def create_option(name, spec): + """Create a click option from a name and a specification.""" + spec = copy.deepcopy(spec) + + is_flag = spec.pop('is_flag', False) + name_dashed = name.replace('_', '-') + option_name = f'--{name_dashed}/--no-{name_dashed}' if is_flag else f'--{name_dashed}' + + kwargs = {'cls': spec.pop('cls', InteractiveOption), 'show_default': True, 'is_flag': is_flag, **spec} + + return click.option(option_name, **kwargs) diff --git a/aiida/cmdline/groups/verdi.py b/aiida/cmdline/groups/verdi.py new file mode 100644 index 0000000000..2937389cd0 --- /dev/null +++ b/aiida/cmdline/groups/verdi.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +"""Subclass of :class:`click.Group` for the ``verdi`` CLI.""" +import base64 +import difflib +import gzip + +import click + +from ..params import options + +__all__ = ('VerdiCommandGroup',) + +GIU = ( + 'ABzY8%U8Kw0{@klyK?I~3`Ki?#qHQ&IIM|J;6yB`9_+{&w)p(JK}vokj-11jhve8xcx?dZ>+9nwrEF!x*S>9A+EWYrR?6GA-u?jFa+et65GF@1+D{%' + '8{C~xjt%>uVM4RTSS?j2M)XH%T#>M{K$lE2XGD`RS0T67213wbAs!SZmn+;(-m!>f(T@e%@oxd`yRBp9nu+9N`4xv8AS@O$CaQ;7FXzM=ug^$?3ta2551EDL`wK4|Cm' + '%RnJdS#0UFwVweDkcfdNjtUv1N^iSQui#TL(q!FmIeKb!yW4|L`@!@-4x6' + 'B6I^ptRdH+4o0ODM;1_f^}4@LMe@#_YHz0wQdq@d)@n)uYNtAb2OLo&fpBkct5{~3kbRag^_5QG%qrTksHMXAYAQoz1#2wtHCy0}h?CJtzv&@Q?^9r' + 'd&02;isB7NJMMr7F@>$!ELj(sbwzIR4)rnch=oVZrG;8)%R6}FUk*fv2O&!#ZA)$HloK9!es&4Eb+h=OIyWFha(8PPy9u?NqfkuPYg;GO1RVzBLX)7' + 'ORMM>1hEM`-96mGjJ+A!e-_}4X{M|4CkKE~uF4j+LW#6IsFa*_da_mLqzr)E<`%ikthkMO2>65cNMtpDE*VejqZV^MyewPJJAS*VM6jY;QY' + '#g7gOKgPbFg{@;YDL6Gbxxr|2T&BQunB?PBetq?X>jW1hFF7&>EaYkKYqIa_ld(Z@AJT' + '+lJ(Pd;+?<&&M>A0agti19^z3n4Z6_WG}c~_+XHyJI_iau7+V$#YA$pJ~H)yHEVy1D?5^Sw`tb@{nnNNo=eSMZLf0>m^A@7f{y$nb_HJWgLRtZ?x2?*>SwM?JoQ>p|-1ZRU0#+{^UhK22+~o' + 'R9k7rh(GH9y|jm){jY9_xAI4N_EfU#4' + 'taTUXFY4a4l$v=N-+f+w&wuH;Z(6p6#=n8XwlZ;*L&-rcL~T_vEm@#-Xi8&g06!MO+R( has plugin type "{}" while "{}" is required'.format( diff --git a/aiida/engine/daemon/execmanager.py b/aiida/engine/daemon/execmanager.py index 2a87c5e6e2..ab4236d70b 100644 --- a/aiida/engine/daemon/execmanager.py +++ b/aiida/engine/daemon/execmanager.py @@ -28,7 +28,7 @@ from aiida.common.folders import SandboxFolder from aiida.common.links import LinkType from aiida.manage.configuration import get_config_option -from aiida.orm import CalcJobNode, Code, FolderData, Node, RemoteData, load_node +from aiida.orm import CalcJobNode, Code, FolderData, Node, PortableCode, RemoteData, load_node from aiida.orm.utils.log import get_dblogger_extra from aiida.repository.common import FileType from aiida.schedulers.datastructures import JobState @@ -174,7 +174,7 @@ def upload_calculation( # Still, beware! The code file itself could be overwritten... # But I checked for this earlier. for code in input_codes: - if code.is_local(): + if isinstance(code, PortableCode): # Note: this will possibly overwrite files for filename in code.base.repository.list_object_names(): # Note, once #2579 is implemented, use the `node.open` method instead of the named temporary file in @@ -184,7 +184,7 @@ def upload_calculation( handle.write(code.base.repository.get_object_content(filename, mode='rb')) handle.flush() transport.put(handle.name, filename) - transport.chmod(code.get_local_executable(), 0o755) # rwxr-xr-x + transport.chmod(code.filepath_executable, 0o755) # rwxr-xr-x # local_copy_list is a list of tuples, each with (uuid, dest_path, rel_path) # NOTE: validation of these lists are done inside calculation.presubmit() diff --git a/aiida/engine/processes/calcjobs/calcjob.py b/aiida/engine/processes/calcjobs/calcjob.py index 9d71583374..3b07181678 100644 --- a/aiida/engine/processes/calcjobs/calcjob.py +++ b/aiida/engine/processes/calcjobs/calcjob.py @@ -194,7 +194,7 @@ def define(cls, spec: CalcJobProcessSpec) -> None: # type: ignore[override] # yapf: disable super().define(spec) spec.inputs.validator = validate_calc_job # type: ignore[assignment] # takes only PortNamespace not Port - spec.input('code', valid_type=orm.Code, required=False, + spec.input('code', valid_type=orm.AbstractCode, required=False, help='The `Code` to use for this job. This input is required, unless the `remote_folder` input is ' 'specified, which means an existing job is being imported and no code will actually be run.') spec.input('remote_folder', valid_type=orm.RemoteData, required=False, @@ -592,7 +592,7 @@ def presubmit(self, folder: Folder) -> CalcInfo: from aiida.common.datastructures import CodeInfo, CodeRunMode from aiida.common.exceptions import InputValidationError, InvalidOperation, PluginInternalError, ValidationError from aiida.common.utils import validate_list_of_string_tuples - from aiida.orm import Code, Computer, load_node + from aiida.orm import AbstractCode, Code, Computer, InstalledCode, PortableCode, load_node from aiida.schedulers.datastructures import JobTemplate, JobTemplateCodeInfo inputs = self.node.base.links.get_incoming(link_type=LinkType.INPUT_CALC) @@ -601,19 +601,19 @@ def presubmit(self, folder: Folder) -> CalcInfo: raise InvalidOperation('calculation node is not stored.') computer = self.node.computer - codes = [_ for _ in inputs.all_nodes() if isinstance(_, Code)] + codes = [_ for _ in inputs.all_nodes() if isinstance(_, AbstractCode)] for code in codes: - if not code.can_run_on(computer): + if not code.can_run_on_computer(computer): raise InputValidationError( 'The selected code {} for calculation {} cannot run on computer {}'.format( code.pk, self.node.pk, computer.label ) ) - if code.is_local() and code.get_local_executable() in folder.get_content_list(): + if isinstance(code, PortableCode) and str(code.filepath_executable) in folder.get_content_list(): raise PluginInternalError( - f'The plugin created a file {code.get_local_executable()} that is also the executable name!' + f'The plugin created a file {code.filepath_executable} that is also the executable name!' ) calc_info = self.prepare_for_submission(folder) @@ -663,12 +663,12 @@ def presubmit(self, folder: Folder) -> CalcInfo: # would return None, in which case the join method would raise # an exception prepend_texts = [computer.get_prepend_text()] + \ - [code.get_prepend_text() for code in codes] + \ + [code.prepend_text for code in codes] + \ [calc_info.prepend_text, self.node.get_option('prepend_text')] job_tmpl.prepend_text = '\n\n'.join(prepend_text for prepend_text in prepend_texts if prepend_text) append_texts = [self.node.get_option('append_text'), calc_info.append_text] + \ - [code.get_append_text() for code in codes] + \ + [code.append_text for code in codes] + \ [computer.get_append_text()] job_tmpl.append_text = '\n\n'.join(append_text for append_text in append_texts if append_text) @@ -696,7 +696,7 @@ def presubmit(self, folder: Folder) -> CalcInfo: if code_info.code_uuid is None: raise PluginInternalError('CalcInfo should have the information of the code to be launched') - this_code = load_node(code_info.code_uuid, sub_classes=(Code,)) + this_code = load_node(code_info.code_uuid, sub_classes=(Code, InstalledCode, PortableCode)) # To determine whether this code should be run with MPI enabled, we get the value that was set in the inputs # of the entire process, which can then be overwritten by the value from the `CodeInfo`. This allows plugins @@ -715,12 +715,12 @@ def presubmit(self, folder: Folder) -> CalcInfo: else: prepend_cmdline_params = [] - cmdline_params = [this_code.get_execname()] + (code_info.cmdline_params or []) + cmdline_params = [str(this_code.get_executable())] + (code_info.cmdline_params or []) tmpl_code_info = JobTemplateCodeInfo() tmpl_code_info.prepend_cmdline_params = prepend_cmdline_params tmpl_code_info.cmdline_params = cmdline_params - tmpl_code_info.use_double_quotes = [computer.get_use_double_quotes(), this_code.get_use_double_quotes()] + tmpl_code_info.use_double_quotes = [computer.get_use_double_quotes(), this_code.use_double_quotes] tmpl_code_info.stdin_name = code_info.stdin_name tmpl_code_info.stdout_name = code_info.stdout_name tmpl_code_info.stderr_name = code_info.stderr_name diff --git a/aiida/engine/processes/process.py b/aiida/engine/processes/process.py index 8b98ec6a9d..cbe20755b7 100644 --- a/aiida/engine/processes/process.py +++ b/aiida/engine/processes/process.py @@ -713,8 +713,8 @@ def _setup_inputs(self) -> None: continue # Special exception: set computer if node is a remote Code and our node does not yet have a computer set - if isinstance(node, orm.Code) and not node.is_local() and not self.node.computer: - self.node.computer = node.get_remote_computer() + if isinstance(node, orm.InstalledCode) and not self.node.computer: + self.node.computer = node.computer # Need this special case for tests that use ProcessNodes as classes if isinstance(self.node, orm.CalculationNode): diff --git a/aiida/manage/tests/pytest_fixtures.py b/aiida/manage/tests/pytest_fixtures.py index 9173ff67e0..b4ea2fba11 100644 --- a/aiida/manage/tests/pytest_fixtures.py +++ b/aiida/manage/tests/pytest_fixtures.py @@ -199,13 +199,18 @@ def get_code(entry_point, executable, computer=aiida_localhost, label=None, prep :rtype: :py:class:`aiida.orm.Code` """ from aiida.common import exceptions - from aiida.orm import Code, Computer, QueryBuilder + from aiida.orm import Computer, InstalledCode, QueryBuilder if label is None: label = executable builder = QueryBuilder().append(Computer, filters={'uuid': computer.uuid}, tag='computer') - builder.append(Code, filters={'label': label, 'attributes.input_plugin': entry_point}, with_computer='computer') + builder.append( + InstalledCode, filters={ + 'label': label, + 'attributes.input_plugin': entry_point + }, with_computer='computer' + ) try: code = builder.one()[0] @@ -218,15 +223,19 @@ def get_code(entry_point, executable, computer=aiida_localhost, label=None, prep if not executable_path: raise ValueError(f'The executable "{executable}" was not found in the $PATH.') - code = Code(input_plugin_name=entry_point, remote_computer_exec=[computer, executable_path]) - code.label = label - code.description = label + code = InstalledCode( + label=label, + description=label, + default_calc_job_plugin=entry_point, + computer=computer, + filepath_executable=executable_path + ) if prepend_text is not None: - code.set_prepend_text(prepend_text) + code.prepend_text = prepend_text if append_text is not None: - code.set_append_text(append_text) + code.append_text = append_text return code.store() diff --git a/aiida/orm/__init__.py b/aiida/orm/__init__.py index db3d2bf518..1821d65103 100644 --- a/aiida/orm/__init__.py +++ b/aiida/orm/__init__.py @@ -28,6 +28,7 @@ __all__ = ( 'ASCENDING', + 'AbstractCode', 'AbstractNodeMeta', 'ArrayData', 'AttributeManager', @@ -60,6 +61,7 @@ 'Group', 'GroupEntityLoader', 'ImportGroup', + 'InstalledCode', 'Int', 'JsonableData', 'Kind', @@ -78,6 +80,7 @@ 'OrbitalData', 'OrderSpecifier', 'OrmEntityLoader', + 'PortableCode', 'ProcessNode', 'ProjectionData', 'QueryBuilder', diff --git a/aiida/orm/nodes/__init__.py b/aiida/orm/nodes/__init__.py index f192d2fafd..5ba5f867fc 100644 --- a/aiida/orm/nodes/__init__.py +++ b/aiida/orm/nodes/__init__.py @@ -21,6 +21,7 @@ from .repository import * __all__ = ( + 'AbstractCode', 'ArrayData', 'BandsData', 'BaseType', @@ -35,6 +36,7 @@ 'EnumData', 'Float', 'FolderData', + 'InstalledCode', 'Int', 'JsonableData', 'Kind', @@ -45,6 +47,7 @@ 'NodeRepository', 'NumericType', 'OrbitalData', + 'PortableCode', 'ProcessNode', 'ProjectionData', 'RemoteData', diff --git a/aiida/orm/nodes/data/__init__.py b/aiida/orm/nodes/data/__init__.py index df9081ed74..6d5826eccb 100644 --- a/aiida/orm/nodes/data/__init__.py +++ b/aiida/orm/nodes/data/__init__.py @@ -36,6 +36,7 @@ from .upf import * __all__ = ( + 'AbstractCode', 'ArrayData', 'BandsData', 'BaseType', @@ -47,6 +48,7 @@ 'EnumData', 'Float', 'FolderData', + 'InstalledCode', 'Int', 'JsonableData', 'Kind', @@ -54,6 +56,7 @@ 'List', 'NumericType', 'OrbitalData', + 'PortableCode', 'ProjectionData', 'RemoteData', 'RemoteStashData', diff --git a/aiida/orm/nodes/data/code/__init__.py b/aiida/orm/nodes/data/code/__init__.py new file mode 100644 index 0000000000..200466ecbc --- /dev/null +++ b/aiida/orm/nodes/data/code/__init__.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +"""Data plugins that represent an executable code.""" + +# AUTO-GENERATED + +# yapf: disable +# pylint: disable=wildcard-import + +from .abstract import * +from .installed import * +from .legacy import * +from .portable import * + +__all__ = ( + 'AbstractCode', + 'Code', + 'InstalledCode', + 'PortableCode', +) + +# yapf: enable diff --git a/aiida/orm/nodes/data/code/abstract.py b/aiida/orm/nodes/data/code/abstract.py new file mode 100644 index 0000000000..c267e94ac9 --- /dev/null +++ b/aiida/orm/nodes/data/code/abstract.py @@ -0,0 +1,288 @@ +# -*- coding: utf-8 -*- +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +"""Abstract data plugin representing an executable code.""" +from __future__ import annotations + +import abc +import collections + +import click + +from aiida.cmdline.params.options.interactive import TemplateInteractiveOption +from aiida.common import exceptions +from aiida.common.lang import type_check +from aiida.orm import Computer +from aiida.plugins import CalculationFactory + +from ..data import Data + +__all__ = ('AbstractCode',) + + +class AbstractCode(Data, metaclass=abc.ABCMeta): + """Abstract data plugin representing an executable code.""" + + # Should become ``default_calc_job_plugin`` once ``Code`` is dropped in ``aiida-core==3.0`` + _KEY_ATTRIBUTE_DEFAULT_CALC_JOB_PLUGIN: str = 'input_plugin' + _KEY_ATTRIBUTE_APPEND_TEXT: str = 'append_text' + _KEY_ATTRIBUTE_PREPEND_TEXT: str = 'prepend_text' + _KEY_ATTRIBUTE_USE_DOUBLE_QUOTES: str = 'use_double_quotes' + _KEY_EXTRA_IS_HIDDEN: str = 'hidden' # Should become ``is_hidden`` once ``Code`` is dropped + + def __init__( + self, + default_calc_job_plugin: str = None, + append_text: str = '', + prepend_text: str = '', + use_double_quotes: bool = False, + is_hidden: bool = False, + **kwargs + ): + """Construct a new instance. + + :param default_calc_job_plugin: The entry point name of the default ``CalcJob`` plugin to use. + :param append_text: The text that should be appended to the run line in the job script. + :param prepend_text: The text that should be prepended to the run line in the job script. + :param use_double_quotes: Whether the command line invocation of this code should be escaped with double quotes. + :param is_hidden: Whether the code is hidden. + """ + super().__init__(**kwargs) + self.default_calc_job_plugin = default_calc_job_plugin + self.append_text = append_text + self.prepend_text = prepend_text + self.use_double_quotes = use_double_quotes + self.use_double_quotes = use_double_quotes + self.is_hidden = is_hidden + + @abc.abstractmethod + def can_run_on_computer(self, computer: Computer) -> bool: + """Return whether the code can run on a given computer. + + :param computer: The computer. + :return: ``True`` if the code can run on ``computer``, ``False`` otherwise. + """ + + @abc.abstractmethod + def get_executable(self) -> str: + """Return the executable that the submission script should execute to run the code. + + :return: The executable to be called in the submission script. + """ + + @property + @abc.abstractmethod + def full_label(self) -> str: + """Return the full label of this code. + + The full label can be just the label itself but it can be something else. However, it at the very least has to + include the label of the code. + + :return: The full label of the code. + """ + + @Data.label.setter + def label(self, value: str) -> None: + """Set the label. + + The label cannot contain any ``@`` symbols. + + :param value: The new label. + :raises ValueError: If the label contains invalid characters. + """ + type_check(value, str) + + if '@' in value: + raise ValueError('The label contains a `@` symbol, which is not allowed.') + + self.backend_entity.label = value + + @property + def default_calc_job_plugin(self) -> str | None: + """Return the optional default ``CalcJob`` plugin. + + :return: The entry point name of the default ``CalcJob`` plugin to use. + """ + return self.base.attributes.get(self._KEY_ATTRIBUTE_DEFAULT_CALC_JOB_PLUGIN, None) + + @default_calc_job_plugin.setter + def default_calc_job_plugin(self, value: str | None) -> None: + """Set the default ``CalcJob`` plugin. + + :param value: The entry point name of the default ``CalcJob`` plugin to use. + """ + type_check(value, str, allow_none=True) + self.base.attributes.set(self._KEY_ATTRIBUTE_DEFAULT_CALC_JOB_PLUGIN, value) + + @property + def append_text(self) -> str: + """Return the text that should be appended to the run line in the job script. + + :return: The text that should be appended to the run line in the job script. + """ + return self.base.attributes.get(self._KEY_ATTRIBUTE_APPEND_TEXT, '') + + @append_text.setter + def append_text(self, value: str) -> None: + """Set the text that should be appended to the run line in the job script. + + :param value: The text that should be appended to the run line in the job script. + """ + type_check(value, str, allow_none=True) + self.base.attributes.set(self._KEY_ATTRIBUTE_APPEND_TEXT, value) + + @property + def prepend_text(self) -> str: + """Return the text that should be prepended to the run line in the job script. + + :return: The text that should be prepended to the run line in the job script. + """ + return self.base.attributes.get(self._KEY_ATTRIBUTE_PREPEND_TEXT, '') + + @prepend_text.setter + def prepend_text(self, value: str) -> None: + """Set the text that should be prepended to the run line in the job script. + + :param value: The text that should be prepended to the run line in the job script. + """ + type_check(value, str, allow_none=True) + self.base.attributes.set(self._KEY_ATTRIBUTE_PREPEND_TEXT, value) + + @property + def use_double_quotes(self) -> bool: + """Return whether the command line invocation of this code should be escaped with double quotes. + + :return: ``True`` if to escape with double quotes, ``False`` otherwise. + """ + return self.base.attributes.get(self._KEY_ATTRIBUTE_USE_DOUBLE_QUOTES, False) + + @use_double_quotes.setter + def use_double_quotes(self, value: bool) -> None: + """Set whether the command line invocation of this code should be escaped with double quotes. + + :param value: ``True`` if to escape with double quotes, ``False`` otherwise. + """ + type_check(value, bool) + self.base.attributes.set(self._KEY_ATTRIBUTE_USE_DOUBLE_QUOTES, value) + + @property + def is_hidden(self) -> bool: + """Return whether the code is hidden. + + :return: ``True`` if the code is hidden, ``False`` otherwise, which is also the default. + """ + return self.base.extras.get(self._KEY_EXTRA_IS_HIDDEN, False) + + @is_hidden.setter + def is_hidden(self, value: bool) -> None: + """Define whether the code is hidden or not. + + :param value: ``True`` if the code should be hidden, ``False`` otherwise. + """ + type_check(value, bool) + self.base.extras.set(self._KEY_EXTRA_IS_HIDDEN, value) + + def get_builder(self): + """Create and return a new ``ProcessBuilder`` for the ``CalcJob`` class of the plugin configured for this code. + + The configured calculation plugin class is defined by the ``default_calc_job_plugin`` property. + + .. note:: it also sets the ``builder.code`` value. + + :return: a ``ProcessBuilder`` instance with the ``code`` input already populated with ourselves + :raise aiida.common.EntryPointError: if the specified plugin does not exist. + :raise ValueError: if no default plugin was specified. + """ + entry_point = self.default_calc_job_plugin + + if entry_point is None: + raise ValueError('No default calculation input plugin specified for this code') + + try: + process_class = CalculationFactory(entry_point) + except exceptions.EntryPointError: + raise exceptions.EntryPointError(f'The calculation entry point `{entry_point}` could not be loaded') + + builder = process_class.get_builder() + builder.code = self + + return builder + + @staticmethod + def cli_validate_label_uniqueness(_, __, value): + """Validate the uniqueness of the label of the code.""" + from aiida.orm import load_code + + try: + load_code(value) + except exceptions.NotExistent: + pass + except exceptions.MultipleObjectsError: + raise click.BadParameter(f'Multiple codes with the label `{value}` already exist.') + else: + raise click.BadParameter(f'A code with the label `{value}` already exists.') + + return value + + @classmethod + def get_cli_options(cls) -> collections.OrderedDict: + """Return the CLI options that would allow to create an instance of this class.""" + return collections.OrderedDict(cls._get_cli_options()) + + @classmethod + def _get_cli_options(cls) -> dict: + """Return the CLI options that would allow to create an instance of this class.""" + return { + 'label': { + 'required': True, + 'type': click.STRING, + 'prompt': 'Label', + 'help': 'A unique label to identify the code by.', + 'callback': cls.cli_validate_label_uniqueness, + }, + 'description': { + 'type': click.STRING, + 'prompt': 'Description', + 'help': 'Human-readable description of this code ideally including version and compilation environment.' + }, + 'default_calc_job_plugin': { + 'type': click.STRING, + 'prompt': 'Default `CalcJob` plugin', + 'help': 'Entry point name of the default plugin (as listed in `verdi plugin list aiida.calculations`).' + }, + 'prepend_text': { + 'cls': TemplateInteractiveOption, + 'type': click.STRING, + 'default': '', + 'prompt': 'Prepend script', + 'help': 'Bash commands that should be prepended to the run line in all submit scripts for this code.', + 'extension': '.bash', + 'header': 'PREPEND_TEXT: if there is any bash commands that should be prepended to the executable call ' + 'in all submit scripts for this code, type that between the equal signs below and save the file.', + 'footer': 'All lines that start with `#=`: will be ignored.' + }, + 'append_text': { + 'cls': TemplateInteractiveOption, + 'type': click.STRING, + 'default': '', + 'prompt': 'Append script', + 'help': 'Bash commands that should be appended to the run line in all submit scripts for this code.', + 'extension': '.bash', + 'header': 'APPEND_TEXT: if there is any bash commands that should be appended to the executable call ' + 'in all submit scripts for this code, type that between the equal signs below and save the file.', + 'footer': 'All lines that start with `#=`: will be ignored.' + }, + 'use_double_quotes': { + 'is_flag': True, + 'default': False, + 'help': 'Whether the executable and arguments of the code in the submission script should be escaped ' + 'with single or double quotes.', + 'prompt': 'Escape using double quotes', + }, + } diff --git a/aiida/orm/nodes/data/code/installed.py b/aiida/orm/nodes/data/code/installed.py new file mode 100644 index 0000000000..ea4f26f235 --- /dev/null +++ b/aiida/orm/nodes/data/code/installed.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +"""Data plugin representing an executable code on a remote computer. + +This plugin should be used if an executable is pre-installed on a computer. The ``InstalledCode`` represents the code by +storing the absolute filepath of the relevant executable and the computer on which it is installed. The computer is +represented by an instance of :class:`aiida.orm.computers.Computer`. Each time a :class:`aiida.engine.CalcJob` is run +using an ``InstalledCode``, it will run its executable on the associated computer. +""" +import pathlib + +import click + +from aiida.cmdline.params.types import ComputerParamType +from aiida.common import exceptions +from aiida.common.lang import type_check +from aiida.common.log import override_log_level +from aiida.orm import Computer + +from .abstract import AbstractCode + +__all__ = ('InstalledCode',) + + +class InstalledCode(AbstractCode): + """Data plugin representing an executable code on a remote computer.""" + + _KEY_ATTRIBUTE_FILEPATH_EXECUTABLE: str = 'filepath_executable' + + def __init__(self, computer: Computer, filepath_executable: str, **kwargs): + """Construct a new instance. + + :param computer: The remote computer on which the executable is located. + :param filepath_executable: The absolute filepath of the executable on the remote computer. + """ + super().__init__(**kwargs) + self.computer = computer + self.filepath_executable = filepath_executable + + def _validate(self): + """Validate the instance by checking that a computer has been defined. + + :raises :class:`aiida.common.exceptions.ValidationError`: If the state of the node is invalid. + """ + super()._validate() + + if not self.computer: + raise exceptions.ValidationError('The `computer` is undefined.') + + try: + self.filepath_executable + except TypeError as exception: + raise exceptions.ValidationError('The `filepath_executable` is not set.') from exception + + def validate_filepath_executable(self): + """Validate the ``filepath_executable`` attribute. + + Checks whether the executable exists on the remote computer if a transport can be opened to it. This method + is intentionally not called in ``_validate`` as to allow the creation of ``Code`` instances whose computers can + not yet be connected to and as to not require the overhead of opening transports in storing a new code. + + :raises `~aiida.common.exceptions.ValidationError`: if no transport could be opened or if the defined executable + does not exist on the remote computer. + """ + try: + with override_log_level(): # Temporarily suppress noisy logging + with self.computer.get_transport() as transport: + file_exists = transport.isfile(self.filepath_executable) + except Exception: # pylint: disable=broad-except + raise exceptions.ValidationError( + 'Could not connect to the configured computer to determine whether the specified executable exists.' + ) + + if not file_exists: + raise exceptions.ValidationError( + f'The provided remote absolute path `{self.filepath_executable}` does not exist on the computer.' + ) + + def can_run_on_computer(self, computer: Computer) -> bool: + """Return whether the code can run on a given computer. + + :param computer: The computer. + :return: ``True`` if the provided computer is the same as the one configured for this code. + """ + type_check(computer, Computer) + return computer.pk == self.computer.pk + + def get_executable(self) -> str: + """Return the executable that the submission script should execute to run the code. + + :return: The executable to be called in the submission script. + """ + return self.filepath_executable + + @property + def full_label(self) -> str: + """Return the full label of this code. + + The full label can be just the label itself but it can be something else. However, it at the very least has to + include the label of the code. + + :return: The full label of the code. + """ + return f'{self.label}@{self.computer.label}' + + @property + def filepath_executable(self) -> pathlib.PurePath: + """Return the absolute filepath of the executable that this code represents. + + :return: The absolute filepath of the executable. + """ + return pathlib.PurePath(self.base.attributes.get(self._KEY_ATTRIBUTE_FILEPATH_EXECUTABLE)) + + @filepath_executable.setter + def filepath_executable(self, value: str) -> None: + """Set the absolute filepath of the executable that this code represents. + + :param value: The absolute filepath of the executable. + """ + type_check(value, str) + + if not pathlib.PurePath(value).is_absolute(): + raise ValueError('the `filepath_executable` should be absolute.') + + self.base.attributes.set(self._KEY_ATTRIBUTE_FILEPATH_EXECUTABLE, value) + + @staticmethod + def cli_validate_label_uniqueness(ctx, _, value): + """Validate the uniqueness of the label of the code.""" + from aiida.orm import load_code + + computer = ctx.params.get('computer', None) + + if computer is None: + return value + + full_label = f'{value}@{computer.label}' + + try: + load_code(full_label) + except exceptions.NotExistent: + pass + except exceptions.MultipleObjectsError: + raise click.BadParameter(f'Multiple codes with the label `{full_label}` already exist.') + else: + raise click.BadParameter(f'A code with the label `{full_label}` already exists.') + + return value + + @classmethod + def _get_cli_options(cls) -> dict: + """Return the CLI options that would allow to create an instance of this class.""" + options = { + 'computer': { + 'required': True, + 'prompt': 'Computer', + 'help': 'The remote computer on which the executable resides.', + 'type': ComputerParamType(), + }, + 'filepath_executable': { + 'required': True, + 'type': click.Path(exists=False), + 'prompt': 'Absolute filepath executable', + 'help': 'Absolute filepath of the executable on the remote computer.', + } + } + options.update(**super()._get_cli_options()) + + return options diff --git a/aiida/orm/nodes/data/code.py b/aiida/orm/nodes/data/code/legacy.py similarity index 78% rename from aiida/orm/nodes/data/code.py rename to aiida/orm/nodes/data/code/legacy.py index d5981eb734..4f8961fabb 100644 --- a/aiida/orm/nodes/data/code.py +++ b/aiida/orm/nodes/data/code/legacy.py @@ -12,13 +12,15 @@ from aiida.common import exceptions from aiida.common.log import override_log_level +from aiida.common.warnings import warn_deprecation +from aiida.orm import Computer -from .data import Data +from .abstract import AbstractCode __all__ = ('Code',) -class Code(Data): +class Code(AbstractCode): """ A code entity. It can either be 'local', or 'remote'. @@ -44,9 +46,13 @@ def __init__(self, remote_computer_exec=None, local_executable=None, input_plugi raise ValueError('cannot set `remote_computer_exec` and `local_executable` at the same time') if remote_computer_exec: + warn_deprecation( + 'The `Code` plugin is deprecated, use the `InstalledCode` (`core.code.remote`) instead.', 3 + ) self.set_remote_computer_exec(remote_computer_exec) if local_executable: + warn_deprecation('The `Code` plugin is deprecated, use the `PortableCode` (`core.code.local`) instead.', 3) self.set_local_executable(local_executable) if input_plugin_name: @@ -57,25 +63,53 @@ def __init__(self, remote_computer_exec=None, local_executable=None, input_plugi HIDDEN_KEY = 'hidden' + def can_run_on_computer(self, computer: Computer) -> bool: + """Return whether the code can run on a given computer. + + :param computer: The computer. + :return: ``True`` if the code can run on ``computer``, ``False`` otherwise. + """ + from aiida import orm + from aiida.common.lang import type_check + + if self.is_local(): + return True + + type_check(computer, orm.Computer) + return computer.pk == self.get_remote_computer().pk + + def get_executable(self) -> str: + """Return the executable that the submission script should execute to run the code. + + :return: The executable to be called in the submission script. + """ + if self.is_local(): + return f'./{self.get_local_executable()}' + + return self.get_remote_exec_path() + def hide(self): """ Hide the code (prevents from showing it in the verdi code list) """ - self.base.extras.set(self.HIDDEN_KEY, True) + warn_deprecation('This property is deprecated, use the `Code.is_hidden` property instead.', version=3) + self.is_hidden = True def reveal(self): """ Reveal the code (allows to show it in the verdi code list) By default, it is revealed """ - self.base.extras.set(self.HIDDEN_KEY, False) + warn_deprecation('This property is deprecated, use the `Code.is_hidden` property instead.', version=3) + self.is_hidden = False @property def hidden(self): """ Determines whether the Code is hidden or not """ - return self.base.extras.get(self.HIDDEN_KEY, False) + warn_deprecation('This property is deprecated, use the `Code.is_hidden` property instead.', version=3) + return self.is_hidden def set_files(self, files): """ @@ -102,6 +136,9 @@ def __str__(self): def get_computer_label(self): """Get label of this code's computer.""" + warn_deprecation( + 'This method is deprecated, use the `InstalledCode.computer.label` property instead.', version=3 + ) return 'repository' if self.is_local() else self.computer.label @property @@ -110,33 +147,14 @@ def full_label(self): Returns label of the form @. """ - return f'{self.label}@{self.get_computer_label()}' - - @property - def label(self): - """Return the node label. - - :return: the label - """ - return super().label - - @label.setter - def label(self, value): - """Set the label. - - :param value: the new value to set - """ - if '@' in str(value): - msg = "Code labels must not contain the '@' symbol" - raise ValueError(msg) - - super(Code, self.__class__).label.fset(self, value) # pylint: disable=no-member + return f'{self.label}@{"repository" if self.is_local() else self.computer.label}' def relabel(self, new_label): """Relabel this code. :param new_label: new code label """ + warn_deprecation('This method is deprecated, use the `label` property instead.', version=3) # pylint: disable=unused-argument suffix = f'@{self.computer.label}' if new_label.endswith(suffix): @@ -149,6 +167,7 @@ def get_description(self): :return: string description of this Code instance """ + warn_deprecation('This method is deprecated, use the `description` property instead.', version=3) return f'{self.description}' @classmethod @@ -162,9 +181,10 @@ def get_code_helper(cls, label, machinename=None, backend=None): a code """ from aiida.common.exceptions import MultipleObjectsError, NotExistent - from aiida.orm.computers import Computer from aiida.orm.querybuilder import QueryBuilder + warn_deprecation('This method is deprecated, use `aiida.orm.load_code` instead.', version=3) + query = QueryBuilder(backend=backend) query.append(cls, filters={'label': label}, project='*', tag='code') if machinename: @@ -198,6 +218,8 @@ def get(cls, pk=None, label=None, machinename=None): # pylint: disable=arguments-differ from aiida.orm.utils import load_code + warn_deprecation('This method is deprecated, use `aiida.orm.load_code` instead.', version=3) + # first check if code pk is provided if pk: code_int = int(pk) @@ -237,6 +259,8 @@ def get_from_string(cls, code_string): """ from aiida.common.exceptions import MultipleObjectsError, NotExistent + warn_deprecation('This method is deprecated, use `aiida.orm.load_code` instead.', version=3) + try: label, _, machinename = code_string.partition('@') except AttributeError: @@ -261,6 +285,9 @@ def list_for_plugin(cls, plugin, labels=True, backend=None): otherwise a list of integers with the code PKs. """ from aiida.orm.querybuilder import QueryBuilder + + warn_deprecation('This method has been deprecated, there is no replacement.', version=3) + query = QueryBuilder(backend=backend) query.append(cls, filters={'attributes.input_plugin': {'==': plugin}}) valid_codes = query.all(flat=True) @@ -304,6 +331,10 @@ def validate_remote_exec_path(self): :raises `~aiida.common.exceptions.ValidationError`: if no transport could be opened or if the defined executable does not exist on the remote computer. """ + warn_deprecation( + 'This method is deprecated, use the `InstalledCode.validate_filepath_executable` property instead.', + version=3 + ) filepath = self.get_remote_exec_path() try: @@ -325,71 +356,77 @@ def set_prepend_text(self, code): Pass a string of code that will be put in the scheduler script before the execution of the code. """ - self.base.attributes.set('prepend_text', str(code)) + warn_deprecation('This method is deprecated, use the `prepend_text` property instead.', version=3) + self.prepend_text = code def get_prepend_text(self): """ Return the code that will be put in the scheduler script before the execution, or an empty string if no pre-exec code was defined. """ - return self.base.attributes.get('prepend_text', '') + warn_deprecation('This method is deprecated, use the `prepend_text` property instead.', version=3) + return self.prepend_text def set_input_plugin_name(self, input_plugin): """ Set the name of the default input plugin, to be used for the automatic generation of a new calculation. """ - if input_plugin is None: - self.base.attributes.set('input_plugin', None) - else: - self.base.attributes.set('input_plugin', str(input_plugin)) + warn_deprecation('This method is deprecated, use the `default_calc_job_plugin` property instead.', version=3) + self.default_calc_job_plugin = input_plugin def get_input_plugin_name(self): """ Return the name of the default input plugin (or None if no input plugin was set. """ - return self.base.attributes.get('input_plugin', None) + warn_deprecation('This method is deprecated, use the `default_calc_job_plugin` property instead.', version=3) + return self.default_calc_job_plugin def set_use_double_quotes(self, use_double_quotes: bool): """Set whether the command line invocation of this code should be escaped with double quotes. :param use_double_quotes: True if to escape with double quotes, False otherwise. """ - from aiida.common.lang import type_check - type_check(use_double_quotes, bool) - self.base.attributes.set('use_double_quotes', use_double_quotes) + warn_deprecation('This method is deprecated, use the `use_double_quotes` property instead.', version=3) + self.use_double_quotes = use_double_quotes def get_use_double_quotes(self) -> bool: """Return whether the command line invocation of this code should be escaped with double quotes. :returns: True if to escape with double quotes, False otherwise which is also the default. """ - return self.base.attributes.get('use_double_quotes', False) + warn_deprecation('This method is deprecated, use the `use_double_quotes` property instead.', version=3) + return self.use_double_quotes def set_append_text(self, code): """ Pass a string of code that will be put in the scheduler script after the execution of the code. """ - self.base.attributes.set('append_text', str(code)) + warn_deprecation('This method is deprecated, use the `append_text` property instead.', version=3) + self.append_text = code def get_append_text(self): """ Return the postexec_code, or an empty string if no post-exec code was defined. """ - return self.base.attributes.get('append_text', '') + warn_deprecation('This method is deprecated, use the `append_text` property instead.', version=3) + return self.append_text def set_local_executable(self, exec_name): """ Set the filename of the local executable. Implicitly set the code as local. """ + warn_deprecation('This method is deprecated, use `PortableCode`.', version=3) + self._set_local() - self.base.attributes.set('local_executable', exec_name) + self.filepath_executable = exec_name def get_local_executable(self): - return self.base.attributes.get('local_executable', '') + warn_deprecation('This method is deprecated, use `PortableCode.filepath_executable` instead.', version=3) + return self.filepath_executable def set_remote_computer_exec(self, remote_computer_exec): """ @@ -402,6 +439,8 @@ def set_remote_computer_exec(self, remote_computer_exec): from aiida import orm from aiida.common.lang import type_check + warn_deprecation('This method is deprecated, use `InstalledCode`.', version=3) + if (not isinstance(remote_computer_exec, (list, tuple)) or len(remote_computer_exec) != 2): raise ValueError( 'remote_computer_exec must be a list or tuple of length 2, with machine and executable name' @@ -419,11 +458,14 @@ def set_remote_computer_exec(self, remote_computer_exec): self.base.attributes.set('remote_exec_path', remote_exec_path) def get_remote_exec_path(self): + warn_deprecation('This method is deprecated, use `InstalledCode.filepath_executable` instead.', version=3) if self.is_local(): raise ValueError('The code is local') return self.base.attributes.get('remote_exec_path', '') def get_remote_computer(self): + """Return the remote computer associated with this code.""" + warn_deprecation('This method is deprecated, use the `computer` attribute instead.', version=3) if self.is_local(): raise ValueError('The code is local') @@ -463,6 +505,7 @@ def is_local(self): Return True if the code is 'local', False if it is 'remote' (see also documentation of the set_local and set_remote functions). """ + warn_deprecation('This method is deprecated, use a `PortableCode` instance and check the type.', version=3) return self.base.attributes.get('is_local', None) def can_run_on(self, computer): @@ -477,6 +520,8 @@ def can_run_on(self, computer): from aiida import orm from aiida.common.lang import type_check + warn_deprecation('This method is deprecated, use `can_run_on_computer` instead.', version=3) + if self.is_local(): return True @@ -489,35 +534,9 @@ def get_execname(self): For local codes, it is ./LOCAL_EXECUTABLE_NAME For remote codes, it is the absolute path to the executable. """ + warn_deprecation('This method is deprecated, use `get_executable` instead.', version=3) + if self.is_local(): return f'./{self.get_local_executable()}' return self.get_remote_exec_path() - - def get_builder(self): - """Create and return a new `ProcessBuilder` for the `CalcJob` class of the plugin configured for this code. - - The configured calculation plugin class is defined by the `get_input_plugin_name` method. - - .. note:: it also sets the ``builder.code`` value. - - :return: a `ProcessBuilder` instance with the `code` input already populated with ourselves - :raise aiida.common.EntryPointError: if the specified plugin does not exist. - :raise ValueError: if no default plugin was specified. - """ - from aiida.plugins import CalculationFactory - - plugin_name = self.get_input_plugin_name() - - if plugin_name is None: - raise ValueError('no default calculation input plugin specified for this code') - - try: - process_class = CalculationFactory(plugin_name) - except exceptions.EntryPointError: - raise exceptions.EntryPointError(f'the calculation entry point `{plugin_name}` could not be loaded') - - builder = process_class.get_builder() - builder.code = self - - return builder diff --git a/aiida/orm/nodes/data/code/portable.py b/aiida/orm/nodes/data/code/portable.py new file mode 100644 index 0000000000..613e76bc4b --- /dev/null +++ b/aiida/orm/nodes/data/code/portable.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +"""Data plugin representing an executable code stored in AiiDA's storage. + +This plugin should be used for executables that are not already installed on the target computer, but instead are +available on the machine where AiiDA is running. The plugin assumes that the code is self-contained by a single +directory containing all the necessary files, including a main executable. When constructing a ``PortableCode``, passing +the absolute filepath as ``filepath_files`` will make sure that all the files contained within are uploaded to AiiDA's +storage. The ``filepath_executable`` should indicate the filename of the executable within that directory. Each time a +:class:`aiida.engine.CalcJob` is run using a ``PortableCode``, the uploaded files will be automatically copied to the +working directory on the selected computer and the executable will be run there. +""" +import pathlib + +import click + +from aiida.common import exceptions +from aiida.common.lang import type_check +from aiida.orm import Computer + +from .abstract import AbstractCode + +__all__ = ('PortableCode',) + + +class PortableCode(AbstractCode): + """Data plugin representing an executable code stored in AiiDA's storage.""" + + _KEY_ATTRIBUTE_FILEPATH_EXECUTABLE: str = 'filepath_executable' + + def __init__(self, filepath_executable: str, filepath_files: pathlib.Path, **kwargs): + """Construct a new instance. + + .. note:: If the files necessary for this code are not all located in a single directory or the directory + contains files that should not be uploaded, and so the ``filepath_files`` cannot be used. One can use the + methods of the :class:`aiida.orm.nodes.repository.NodeRepository` class. This can be accessed through the + ``base.repository`` attribute of the instance after it has been constructed. For example:: + + code = PortableCode(filepath_executable='some_name.exe') + code.put_object_from_file() + code.put_object_from_filelike() + code.put_object_from_tree() + + :param filepath_executable: The relative filepath of the executable within the directory of uploaded files. + :param filepath_files: The filepath to the directory containing all the files of the code. + """ + super().__init__(**kwargs) + type_check(filepath_files, pathlib.Path) + self.filepath_executable = filepath_executable + self.base.repository.put_object_from_tree(filepath_files) + + def _validate(self): + """Validate the instance by checking that an executable is defined and it is part of the repository files. + + :raises :class:`aiida.common.exceptions.ValidationError`: If the state of the node is invalid. + """ + super()._validate() + + try: + filepath_executable = self.filepath_executable + except TypeError as exception: + raise exceptions.ValidationError('The `filepath_executable` is not set.') from exception + + objects = self.base.repository.list_object_names() + + if str(filepath_executable) not in objects: + raise exceptions.ValidationError( + f'The executable `{filepath_executable}` is not one of the uploaded files: {objects}' + ) + + def can_run_on_computer(self, computer: Computer) -> bool: + """Return whether the code can run on a given computer. + + A ``PortableCode`` should be able to be run on any computer in principle. + + :param computer: The computer. + :return: ``True`` if the provided computer is the same as the one configured for this code. + """ + return True + + def get_executable(self) -> str: + """Return the executable that the submission script should execute to run the code. + + :return: The executable to be called in the submission script. + """ + return self.filepath_executable + + @property + def full_label(self) -> str: + """Return the full label of this code. + + The full label can be just the label itself but it can be something else. However, it at the very least has to + include the label of the code. + + :return: The full label of the code. + """ + return self.label + + @property + def filepath_executable(self) -> pathlib.PurePath: + """Return the relative filepath of the executable that this code represents. + + :return: The relative filepath of the executable. + """ + return pathlib.PurePath(self.base.attributes.get(self._KEY_ATTRIBUTE_FILEPATH_EXECUTABLE)) + + @filepath_executable.setter + def filepath_executable(self, value: str) -> None: + """Set the relative filepath of the executable that this code represents. + + :param value: The relative filepath of the executable within the directory of uploaded files. + """ + type_check(value, str) + + if pathlib.PurePath(value).is_absolute(): + raise ValueError('The `filepath_executable` should not be absolute.') + + self.base.attributes.set(self._KEY_ATTRIBUTE_FILEPATH_EXECUTABLE, value) + + @classmethod + def _get_cli_options(cls) -> dict: + """Return the CLI options that would allow to create an instance of this class.""" + options = { + 'filepath_executable': { + 'required': True, + 'type': click.STRING, + 'prompt': 'Relative filepath executable', + 'help': 'Relative filepath of executable with directory of code files.', + }, + 'filepath_files': { + 'required': True, + 'type': click.Path(exists=True, file_okay=False, dir_okay=True, path_type=pathlib.Path), + 'prompt': 'Code directory', + 'help': 'Filepath to directory containing code files.', + } + } + options.update(**super()._get_cli_options()) + + return options diff --git a/aiida/orm/utils/builders/code.py b/aiida/orm/utils/builders/code.py index a583f06652..bd110ffb5b 100644 --- a/aiida/orm/utils/builders/code.py +++ b/aiida/orm/utils/builders/code.py @@ -9,16 +9,21 @@ ########################################################################### """Manage code objects with lazy loading of the db env""" import enum -import os +import pathlib from aiida.cmdline.utils.decorators import with_dbenv from aiida.common.utils import ErrorAccumulator +from aiida.common.warnings import warn_deprecation +from aiida.orm import InstalledCode, PortableCode + +warn_deprecation('This module is deprecated. To create a new code instance, simply use the constructor.', version=3) class CodeBuilder: """Build a code with validation of attribute combinations""" def __init__(self, **kwargs): + """Construct a new instance.""" self._err_acc = ErrorAccumulator(self.CodeValidationError) self._code_spec = {} @@ -41,31 +46,27 @@ def new(self): """Build and return a new code instance (not stored)""" self.validate() - from aiida.orm import Code - # Will be used at the end to check if all keys are known (those that are not None) passed_keys = set(k for k in self._code_spec.keys() if self._code_spec[k] is not None) used = set() if self._get_and_count('code_type', used) == self.CodeType.STORE_AND_UPLOAD: - file_list = [ - os.path.realpath(os.path.join(self.code_folder, f)) - for f in os.listdir(self._get_and_count('code_folder', used)) - ] - code = Code(local_executable=self._get_and_count('code_rel_path', used), files=file_list) + code = PortableCode( + filepath_executable=self._get_and_count('code_rel_path', used), + filepath_files=pathlib.Path(self._get_and_count('code_folder', used)) + ) else: - code = Code( - remote_computer_exec=( - self._get_and_count('computer', used), self._get_and_count('remote_abs_path', used) - ) + code = InstalledCode( + computer=self._get_and_count('computer', used), + filepath_executable=self._get_and_count('remote_abs_path', used) ) code.label = self._get_and_count('label', used) code.description = self._get_and_count('description', used) - code.set_input_plugin_name(self._get_and_count('input_plugin', used)) - code.set_use_double_quotes(self._get_and_count('use_double_quotes', used)) - code.set_prepend_text(self._get_and_count('prepend_text', used)) - code.set_append_text(self._get_and_count('append_text', used)) + code.default_calc_job_plugin = self._get_and_count('input_plugin', used) + code.use_double_quotes = self._get_and_count('use_double_quotes', used) + code.prepend_text = self._get_and_count('prepend_text', used) + code.append_text = self._get_and_count('append_text', used) # Complain if there are keys that are passed but not used if passed_keys - used: @@ -98,19 +99,19 @@ def get_code_spec(code): spec = {} spec['label'] = code.label spec['description'] = code.description - spec['input_plugin'] = code.get_input_plugin_name() - spec['use_double_quotes'] = code.get_use_double_quotes() - spec['prepend_text'] = code.get_prepend_text() - spec['append_text'] = code.get_append_text() + spec['input_plugin'] = code.default_calc_job_plugin + spec['use_double_quotes'] = code.use_double_quotes + spec['prepend_text'] = code.prepend_text + spec['append_text'] = code.append_text - if code.is_local(): + if isinstance(code, PortableCode): spec['code_type'] = CodeBuilder.CodeType.STORE_AND_UPLOAD spec['code_folder'] = code.get_code_folder() spec['code_rel_path'] = code.get_code_rel_path() else: spec['code_type'] = CodeBuilder.CodeType.ON_COMPUTER - spec['computer'] = code.get_remote_computer() - spec['remote_abs_path'] = code.get_remote_exec_path() + spec['computer'] = code.computer + spec['remote_abs_path'] = code.get_executable() return spec diff --git a/aiida/plugins/entry_point.py b/aiida/plugins/entry_point.py index 20a3e9fa6a..71f9bdb48c 100644 --- a/aiida/plugins/entry_point.py +++ b/aiida/plugins/entry_point.py @@ -22,6 +22,8 @@ from aiida.common.exceptions import LoadingEntryPointError, MissingEntryPointError, MultipleEntryPointError from aiida.common.warnings import warn_deprecation +from . import factories + __all__ = ('load_entry_point', 'load_entry_point_from_string', 'parse_entry_point', 'get_entry_points') ENTRY_POINT_GROUP_PREFIX = 'aiida.' @@ -89,6 +91,18 @@ class EntryPointFormat(enum.Enum): 'aiida.workflows': ['arithmetic.multiply_add', 'arithmetic.add_multiply'], } +ENTRY_POINT_GROUP_FACTORY_MAPPING = { + 'aiida.calculations': factories.CalculationFactory, + 'aiida.data': factories.DataFactory, + 'aiida.groups': factories.GroupFactory, + 'aiida.parsers': factories.ParserFactory, + 'aiida.schedulers': factories.SchedulerFactory, + 'aiida.transports': factories.TransportFactory, + 'aiida.tools.dbimporters': factories.DbImporterFactory, + 'aiida.tools.data.orbital': factories.OrbitalFactory, + 'aiida.workflows': factories.WorkflowFactory, +} + def parse_entry_point(group: str, spec: str) -> EntryPoint: """Return an entry point, given its group and spec (as formatted in the setup)""" @@ -105,21 +119,7 @@ def validate_registered_entry_points() -> None: # pylint: disable=invalid-name * The resource's type is incompatible with the entry point group that it is defined in. """ - from . import factories - - factory_mapping = { - 'aiida.calculations': factories.CalculationFactory, - 'aiida.data': factories.DataFactory, - 'aiida.groups': factories.GroupFactory, - 'aiida.parsers': factories.ParserFactory, - 'aiida.schedulers': factories.SchedulerFactory, - 'aiida.transports': factories.TransportFactory, - 'aiida.tools.dbimporters': factories.DbImporterFactory, - 'aiida.tools.data.orbital': factories.OrbitalFactory, - 'aiida.workflows': factories.WorkflowFactory, - } - - for entry_point_group, factory in factory_mapping.items(): + for entry_point_group, factory in ENTRY_POINT_GROUP_FACTORY_MAPPING.items(): entry_points = get_entry_points(entry_point_group) for entry_point in entry_points: factory(entry_point.name) diff --git a/aiida/workflows/arithmetic/multiply_add.py b/aiida/workflows/arithmetic/multiply_add.py index 858979816f..3576162ce1 100644 --- a/aiida/workflows/arithmetic/multiply_add.py +++ b/aiida/workflows/arithmetic/multiply_add.py @@ -11,7 +11,7 @@ # start-marker for docs """Implementation of the MultiplyAddWorkChain for testing and demonstration purposes.""" from aiida.engine import ToContext, WorkChain, calcfunction -from aiida.orm import Code, Int +from aiida.orm import AbstractCode, Int from aiida.plugins.factories import CalculationFactory ArithmeticAddCalculation = CalculationFactory('core.arithmetic.add') @@ -32,7 +32,7 @@ def define(cls, spec): spec.input('x', valid_type=Int) spec.input('y', valid_type=Int) spec.input('z', valid_type=Int) - spec.input('code', valid_type=Code) + spec.input('code', valid_type=AbstractCode) spec.outline( cls.multiply, cls.add, diff --git a/docs/source/conf.py b/docs/source/conf.py index 71d93945d3..a83fd17dee 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -232,7 +232,7 @@ def setup(app: Sphinx): # these are mainly required, because sphinx finds multiple references, # in `aiida.orm`` and `aiida.restapi.translator` autodoc_aliases_typing = { - 'Code': 'aiida.orm.nodes.data.code.Code', + 'Code': 'aiida.orm.nodes.data.code.legacy.Code', 'Computer': 'aiida.orm.computers.Computer', 'Data': 'aiida.orm.nodes.data.data.Data', 'Group': 'aiida.orm.groups.Group', @@ -277,7 +277,7 @@ def setup(app: Sphinx): 'aiida.orm.User': 'aiida.orm.users.User', 'aiida.orm.CalculationNode': 'aiida.orm.nodes.process.calculation.calculation.CalculationNode', 'aiida.orm.CalcJobNode': 'aiida.orm.nodes.process.calculation.calcjob.CalcJobNode', - 'aiida.orm.Code': 'aiida.orm.nodes.data.code.Code', + 'aiida.orm.Code': 'aiida.orm.nodes.data.code.legacy.Code', 'aiida.orm.Data': 'aiida.orm.nodes.data.data.Data', 'aiida.orm.Dict': 'aiida.orm.nodes.data.dict.Dict', 'aiida.orm.ProcessNode': 'aiida.orm.nodes.process.process.ProcessNode', diff --git a/docs/source/developer_guide/core/internals.rst b/docs/source/developer_guide/core/internals.rst index bb13565f09..cd0ab9e15c 100644 --- a/docs/source/developer_guide/core/internals.rst +++ b/docs/source/developer_guide/core/internals.rst @@ -6,7 +6,7 @@ Node ++++ All nodes in an AiiDA provenance graph inherit from the :py:class:`~aiida.orm.nodes.node.Node` class. -Among those are the :py:class:`~aiida.orm.nodes.data.data.Data` class, the :py:class:`~aiida.orm.nodes.process.process.ProcessNode` class representing computations that transform data, and the :py:class:`~aiida.orm.nodes.data.code.Code` class representing executables (and file collections that are used by calculations). +Among those are the :py:class:`~aiida.orm.nodes.data.data.Data` class, the :py:class:`~aiida.orm.nodes.process.process.ProcessNode` class representing computations that transform data, and the :py:class:`~aiida.orm.nodes.data.code.abstract.AbstractCode` class representing executables (and file collections that are used by calculations). Immutability concept diff --git a/docs/source/howto/include/snippets/extend_workflows.py b/docs/source/howto/include/snippets/extend_workflows.py index 7975d0ae8e..568ed51e60 100644 --- a/docs/source/howto/include/snippets/extend_workflows.py +++ b/docs/source/howto/include/snippets/extend_workflows.py @@ -11,7 +11,7 @@ # start-marker for docs """Code snippets for the "How to extend workflows" section.""" from aiida.engine import ToContext, WorkChain, calcfunction -from aiida.orm import Bool, Code, Int +from aiida.orm import AbstractCode, Bool, Int from aiida.plugins.factories import CalculationFactory ArithmeticAddCalculation = CalculationFactory('core.arithmetic.add') @@ -38,7 +38,7 @@ def define(cls, spec): spec.input('x', valid_type=Int) spec.input('y', valid_type=Int) spec.input('z', valid_type=Int) - spec.input('code', valid_type=Code) + spec.input('code', valid_type=AbstractCode) spec.outline( cls.multiply, cls.add, @@ -81,7 +81,7 @@ def define(cls, spec): spec.input('x', valid_type=Int) spec.input('y', valid_type=Int) spec.input('z', valid_type=Int) - spec.input('code', valid_type=Code) + spec.input('code', valid_type=AbstractCode) spec.outline( cls.multiply, cls.add, @@ -126,7 +126,7 @@ def define(cls, spec): spec.input('x', valid_type=Int) spec.input('y', valid_type=Int) spec.input('z', valid_type=Int) - spec.input('code', valid_type=Code) + spec.input('code', valid_type=AbstractCode) spec.outline( cls.multiply_add, cls.is_even, diff --git a/docs/source/howto/include/snippets/plugins/launch.py b/docs/source/howto/include/snippets/plugins/launch.py index 3c17f2aa14..f519047b95 100644 --- a/docs/source/howto/include/snippets/plugins/launch.py +++ b/docs/source/howto/include/snippets/plugins/launch.py @@ -13,7 +13,7 @@ code = orm.load_code('diff@localhost') except NotExistent: # Setting up code via python API (or use "verdi code setup") - code = orm.Code(label='diff', remote_computer_exec=[computer, '/usr/bin/diff'], input_plugin_name='diff-tutorial') + code = orm.InstalledCode(label='diff', computer=computer, filepath_executable='/usr/bin/diff', default_calc_job_plugin='diff-tutorial') # Set up inputs builder = code.get_builder() diff --git a/docs/source/howto/plugin_codes.rst b/docs/source/howto/plugin_codes.rst index f387484af7..d899620890 100644 --- a/docs/source/howto/plugin_codes.rst +++ b/docs/source/howto/plugin_codes.rst @@ -125,7 +125,7 @@ There is no ``return`` statement in ``define``: the ``define`` method directly m .. code-block:: python - spec.input('code', valid_type=orm.Code, help='The `Code` to use for this job.') + spec.input('code', valid_type=orm.AbstractCode, help='The `Code` to use for this job.') .. admonition:: Further reading @@ -614,7 +614,7 @@ Continue with :ref:`how-to:plugins-develop` in order to learn how to quickly cre .. |StructureData| replace:: :py:class:`~aiida.orm.nodes.data.structure.StructureData` .. |RemoteData| replace:: :py:class:`~aiida.orm.RemoteData` .. |Dict| replace:: :py:class:`~aiida.orm.nodes.data.dict.Dict` -.. |Code| replace:: :py:class:`~aiida.orm.Code` +.. |Code| replace:: :py:class:`~aiida.orm.nodes.data.code.abstract.AbstractCode` .. |Parser| replace:: :py:class:`~aiida.parsers.parser.Parser` .. |parse| replace:: :py:class:`~aiida.parsers.parser.Parser.parse` .. |folder| replace:: :py:class:`~aiida.common.folders.Folder` diff --git a/docs/source/howto/workchains_restart.rst b/docs/source/howto/workchains_restart.rst index 0ca636d323..9dda7afdfb 100644 --- a/docs/source/howto/workchains_restart.rst +++ b/docs/source/howto/workchains_restart.rst @@ -107,7 +107,7 @@ Next, as with all work chains, we should *define* its process specification: super().define(spec) spec.input('x', valid_type=(orm.Int, orm.Float), help='The left operand.') spec.input('y', valid_type=(orm.Int, orm.Float), help='The right operand.') - spec.input('code', valid_type=orm.Code, help='The code to use to perform the summation.') + spec.input('code', valid_type=orm.AbstractCode, help='The code to use to perform the summation.') spec.output('sum', valid_type=(orm.Int, orm.Float), help='The sum of the left and right operand.') spec.outline( cls.setup, diff --git a/docs/source/intro/tutorial.md b/docs/source/intro/tutorial.md index 32d9bcb278..592b74b866 100644 --- a/docs/source/intro/tutorial.md +++ b/docs/source/intro/tutorial.md @@ -443,7 +443,7 @@ These steps are provided as methods of the `MultiplyAddWorkChain` class. :tags: ["hide-cell"] from aiida.engine import ToContext, WorkChain, calcfunction -from aiida.orm import Code, Int +from aiida.orm import AbstractCode, Int from aiida.plugins.factories import CalculationFactory ArithmeticAddCalculation = CalculationFactory('core.arithmetic.add') @@ -464,7 +464,7 @@ class MultiplyAddWorkChain(WorkChain): spec.input('x', valid_type=Int) spec.input('y', valid_type=Int) spec.input('z', valid_type=Int) - spec.input('code', valid_type=Code) + spec.input('code', valid_type=AbstractCode) spec.outline( cls.multiply, cls.add, diff --git a/docs/source/nitpick-exceptions b/docs/source/nitpick-exceptions index 95cc3b3a82..612300411f 100644 --- a/docs/source/nitpick-exceptions +++ b/docs/source/nitpick-exceptions @@ -24,6 +24,8 @@ py:class ReturnType py:class SelfType py:class CollectionType py:class EntityType +py:class EntityClsType +py:class ProjectType py:class BackendEntityType py:class BackendNodeType py:class TransactionType diff --git a/docs/source/reference/command_line.rst b/docs/source/reference/command_line.rst index 4a1adab63e..4d2991ea18 100644 --- a/docs/source/reference/command_line.rst +++ b/docs/source/reference/command_line.rst @@ -71,6 +71,7 @@ Below is a list with all available subcommands. --help Show this message and exit. Commands: + create Create a new code. delete Delete a code. duplicate Duplicate a code allowing to change some parameters. hide Hide one or more codes from `verdi code list`. diff --git a/docs/source/topics/calculations/concepts.rst b/docs/source/topics/calculations/concepts.rst index 87d681f796..72807d47f7 100644 --- a/docs/source/topics/calculations/concepts.rst +++ b/docs/source/topics/calculations/concepts.rst @@ -127,7 +127,7 @@ the provenance generated by running the calculation job will look something like The execution of the calculation job is represented in the provenance graph by a process node, i.e. the pink square labeled `C\ :sub:`1`` in :numref:`fig_calculation_jobs_provenance_arithmetic_add`. The integer data nodes ``x`` and ``y`` that were passed as inputs are linked to the calculation job as such, as well as the third input ``code``. This input is required for *all* calculation jobs as it represents the external code that is actually executed. -These code nodes are instances of the :py:class:`~aiida.orm.nodes.data.code.Code` class, which is a sub-class of :py:class:`~aiida.orm.nodes.data.data.Data`. +These code nodes are instances of the :py:class:`~aiida.orm.nodes.data.code.abstract.AbstractCode` class, which is a sub-class of :py:class:`~aiida.orm.nodes.data.data.Data`. This means that code instances are a sort of data node. Its function is to record the path to the executable and some other code related attributes defined during the code setup. @@ -150,7 +150,7 @@ When a calculation job is launched, the engine will take it roughly through the * **Retrieve**: once the job has finished, the engine will retrieve the output files, specified by the calculation plugin and store them in a node attached as an output node to the calculation All of these tasks require the engine to interact with the computer, or machine, that will actually run the external code. -Since the :py:class:`~aiida.orm.nodes.data.code.Code` that is used as an input for the calculation job, which is configured for a specific :py:class:`~aiida.orm.computers.Computer`, the engine knows exactly how to execute all these tasks. +Since the :py:class:`~aiida.orm.nodes.data.code.abstract.AbstractCode` that is used as an input for the calculation job, which is configured for a specific :py:class:`~aiida.orm.computers.Computer`, the engine knows exactly how to execute all these tasks. The ``CalcJob`` implementation itself then is completely independent of the machine the code will be run on. To run the calculation job on a different machine, all you have to do is change the ``code`` input to one that is configured for that machine. If the machine is *not* the localhost, the engine will need a way to connect to the remote machine in order to perform each of the four tasks listed above. diff --git a/docs/source/topics/data_types.rst b/docs/source/topics/data_types.rst index 2beb07147d..860db0dcad 100644 --- a/docs/source/topics/data_types.rst +++ b/docs/source/topics/data_types.rst @@ -40,35 +40,43 @@ Below is a list of the core data types already provided with AiiDA, along with t .. table:: :widths: 20 20 45 45 - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | **Class** | **Entry point** | **Stored in database** | **Stored in repository** | - +===========================================================+========================+===================================================+===================================+ - | :ref:`Int ` | ``core.int`` | The integer value | \\- | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`Float ` | ``core.float`` | The float value | \\- | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`Str ` | ``core.str`` | The string | \\- | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`Bool ` | ``core.bool`` | The boolean value | \\- | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`List ` | ``core.list`` | The complete list | \\- | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`Dict ` | ``core.dict`` | The complete dictionary | \\- | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`EnumData ` | ``core.enum`` | The value, name and the class identifier | \\- | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`JsonableData ` | ``core.jsonable`` | The JSON data and the class identifier | \\- | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`ArrayData ` | ``core.array`` | The array names and corresponding shapes | The array data in ``.npy`` format | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`XyData ` | ``core.array.xy`` | The array names and corresponding shapes | The array data in ``.npy`` format | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`SinglefileData ` | ``core.singlefile`` | The filename | The file | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`FolderData ` | ``core.folder`` | \\- | All files and folders | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ - | :ref:`RemoteData ` | ``core.remote`` | The computer and the absolute path to the folder | All files and folders | - +-----------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | **Class** | **Entry point** | **Stored in database** | **Stored in repository** | + +==============================================================+========================+===================================================+===================================+ + | :ref:`Int ` | ``core.int`` | The integer value | ``-`` | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`Float ` | ``core.float`` | The float value | ``-`` | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`Str ` | ``core.str`` | The string | ``-`` | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`Bool ` | ``core.bool`` | The boolean value | ``-`` | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`List ` | ``core.list`` | The complete list | ``-`` | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`Dict ` | ``core.dict`` | The complete dictionary | ``-`` | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`EnumData ` | ``core.enum`` | The value, name and the class identifier | ``-`` | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`JsonableData ` | ``core.jsonable`` | The JSON data and the class identifier | ``-`` | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`ArrayData ` | ``core.array`` | The array names and corresponding shapes | The array data in ``.npy`` format | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`XyData ` | ``core.array.xy`` | The array names and corresponding shapes | The array data in ``.npy`` format | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`SinglefileData ` | ``core.singlefile`` | The filename | The file | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`FolderData ` | ``core.folder`` | ``-`` | All files and folders | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`RemoteData ` | ``core.remote`` | The computer and the absolute path to the folder | All files and folders | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`AbstractCode ` | ``-`` | Default plugin, append/prepend text | ``-`` | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`Code ` | ``core.code`` | The computer and the executable path | All files and folders | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`InstalledCode ` | ``core.code.installed``| The computer and the executable path | ``-`` | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ + | :ref:`PortableCode ` | ``core.code.portable`` | The relative path of the executable | All files and folders of the code | + +--------------------------------------------------------------+------------------------+---------------------------------------------------+-----------------------------------+ .. _topics:data_types:core:base: @@ -400,11 +408,90 @@ To see the contents of a subdirectory, pass the relative path to the :py:meth:`~ Using the :py:meth:`~aiida.orm.RemoteData.listdir()` method, or any method that retrieves information from the remote computer, opens a connection to the remote computer using its transport type. Their use is strongly discouraged when writing scripts and/or workflows. -.. todo:: - .. _topics:data_types:core:code: +.. _topics:data_types:core:code: + +AbstractCode +------------ + +.. versionadded:: 2.1 + +The :class:`aiida.orm.nodes.data.code.abstract.AbstractCode` class provides the abstract class for objects that represent a "code" that can be executed through a :class:`aiida.engine.processes.calcjobs.calcjob.CalcJob` plugin. +There are currently three implementations of this abstract class: + + * :class:`~aiida.orm.nodes.data.code.legacy.Code` (see :ref:`Code `) + * :class:`~aiida.orm.nodes.data.code.installed.InstalledCode` (see :ref:`InstalledCode `) + * :class:`~aiida.orm.nodes.data.code.portable.PortableCode` (see :ref:`PortableCode `) + + +.. _topics:data_types:core:code:legacy: + +Code +---- + +.. deprecated:: 2.1 + +Historically, there was only one code implementation, the :class:`~aiida.orm.nodes.data.code.legacy.Code`, which implemented two different types of code: + + * An executable pre-installed on a computer, represented by a :class:`~aiida.orm.computers.Computer`. + * A directory containing all code files including an executable which would be uploaded to + +These two types were referred to as "remote" and "local" codes. +However, this nomenclature would lead to confusion as a "remote" code could also refer to an executable on the localhost, i.e., the machine where AiiDA itself runs. +In addition, having two different concepts implemented by a single class led to a unintuitive interface. +Therefore, the ``Code`` class was deprecated in ``aiida-core==2.1`` and replaced by the :ref:`InstallCode ` and :ref:`InstallCode `, respectively. +The ``Code`` class is now deprecated and will be removed in ``aiida-core==3.0``. + + +.. _topics:data_types:core:code:installed: + +InstalledCode +------------- + +.. versionadded:: 2.1 + +The :class:`~aiida.orm.nodes.data.code.installed.InstalledCode` class is an implementation of the :class:`~aiida.orm.nodes.data.code.abstract.AbstractCode` class that represents an executable code on a remote computer. +This plugin should be used if an executable is pre-installed on a computer. +The ``InstalledCode`` represents the code by storing the absolute filepath of the relevant executable and the computer on which it is installed. +The computer is represented by an instance of :class:`~aiida.orm.computers.Computer`. +Each time a :class:`~aiida.engine.CalcJob` is run using an ``InstalledCode``, it will run its executable on the associated computer. +Example of creating an ``InstalledCode``: + +.. code:: python + + from aiida.orm import InstalledCode + code = InstalledCode( + label='some-label', + computer=load_computer('localhost'), + filepath_executable='/usr/bin/bash' + ) + + +.. _topics:data_types:core:code:portable: + +PortableCode +------------ + +.. versionadded:: 2.1 + +The :class:`~aiida.orm.nodes.data.code.portable.PortableCode` class is an implementation of the :class:`~aiida.orm.nodes.data.code.abstract.AbstractCode` class that represents an executable code stored in AiiDA's storage. +This plugin should be used for executables that are not already installed on the target computer, but instead are available on the machine where AiiDA is running. +The plugin assumes that the code is self-contained by a single directory containing all the necessary files, including a main executable. +When constructing a ``PortableCode``, passing the absolute filepath as ``filepath_files`` will make sure that all the files contained within are uploaded to AiiDA's storage. +The ``filepath_executable`` should indicate the filename of the executable within that directory. +Each time a :class:`~aiida.engine.CalcJob` is run using a ``PortableCode``, the uploaded files will be automatically copied to the working directory on the selected computer and the executable will be run there. +Example of creating an ``PortableCode``: + +.. code:: python + + from pathlib import Path + from aiida.orm import PortableCode + code = PortableCode( + label='some-label', + filepath_files=Path('/some/path/code'), + filepath_executable='executable.exe' + ) - title: Code .. _topics:data_types:materials: diff --git a/docs/source/topics/plugins.rst b/docs/source/topics/plugins.rst index 5633b85539..cbb1b1ad41 100644 --- a/docs/source/topics/plugins.rst +++ b/docs/source/topics/plugins.rst @@ -212,7 +212,7 @@ Usage:: ----------------- ``verdi`` uses the `click_` framework, which makes it possible to add new subcommands to existing verdi commands, such as ``verdi data mydata``. -AiiDA expects each entry point to be either a ``click.Command`` or ``click.CommandGroup``. At present extra commands can be injected at the following levels: +AiiDA expects each entry point to be either a ``click.Command`` or ``click.Group``. At present extra commands can be injected at the following levels: * As a :ref:`direct subcommand of verdi data` * As a :ref:`subcommand of verdi data structure import` diff --git a/docs/source/topics/processes/usage.rst b/docs/source/topics/processes/usage.rst index c8c1861be6..3494c13f34 100644 --- a/docs/source/topics/processes/usage.rst +++ b/docs/source/topics/processes/usage.rst @@ -410,7 +410,7 @@ In an interactive shell, you can get this information to display as follows:: "name": "code", "required": "True" "non_db": "False" - "valid_type": "" + "valid_type": "" "help": "The Code to use for this job.", In the ``Docstring`` you will see a ``help`` string that contains more detailed information about the input port. diff --git a/pyproject.toml b/pyproject.toml index 33f9731913..b9ec66c940 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -167,7 +167,9 @@ runaiida = "aiida.cmdline.commands.cmd_run:run" "core.base" = "aiida.orm.nodes.data:BaseType" "core.bool" = "aiida.orm.nodes.data.bool:Bool" "core.cif" = "aiida.orm.nodes.data.cif:CifData" -"core.code" = "aiida.orm.nodes.data.code:Code" +"core.code" = "aiida.orm.nodes.data.code.legacy:Code" +"core.code.portable" = "aiida.orm.nodes.data.code.portable:PortableCode" +"core.code.installed" = "aiida.orm.nodes.data.code.installed:InstalledCode" "core.dict" = "aiida.orm.nodes.data.dict:Dict" "core.enum" = "aiida.orm.nodes.data.enum:EnumData" "core.float" = "aiida.orm.nodes.data.float:Float" diff --git a/tests/benchmark/test_engine.py b/tests/benchmark/test_engine.py index e4cc385033..a99438d565 100644 --- a/tests/benchmark/test_engine.py +++ b/tests/benchmark/test_engine.py @@ -19,7 +19,7 @@ from aiida.engine import WorkChain, run_get_node, submit, while_ from aiida.manage import get_manager -from aiida.orm import Code, Int +from aiida.orm import InstalledCode, Int from aiida.plugins.factories import CalculationFactory ArithmeticAddCalculation = CalculationFactory('core.arithmetic.add') @@ -119,7 +119,9 @@ def run_task(self): @pytest.mark.benchmark(group='engine') def test_workchain_local(benchmark, aiida_localhost, workchain, iterations, outgoing): """Benchmark Workchains, executed in the local runner.""" - code = Code(input_plugin_name='core.arithmetic.add', remote_computer_exec=[aiida_localhost, '/bin/true']) + code = InstalledCode( + default_calc_job_plugin='core.arithmetic.add', computer=aiida_localhost, filepath_executable='/bin/true' + ) def _run(): return run_get_node(workchain, iterations=Int(iterations), code=code) @@ -175,7 +177,9 @@ async def _do_submit(): @pytest.mark.benchmark(group='engine') def test_workchain_daemon(benchmark, submit_get_node, aiida_localhost, workchain, iterations, outgoing): """Benchmark Workchains, executed in the via a daemon runner.""" - code = Code(input_plugin_name='core.arithmetic.add', remote_computer_exec=[aiida_localhost, '/bin/true']) + code = InstalledCode( + default_calc_job_plugin='core.arithmetic.add', computer=aiida_localhost, filepath_executable='/bin/true' + ) def _run(): return submit_get_node(workchain, iterations=Int(iterations), code=code) diff --git a/tests/calculations/arithmetic/test_add.py b/tests/calculations/arithmetic/test_add.py index a78cc45000..fac81f1556 100644 --- a/tests/calculations/arithmetic/test_add.py +++ b/tests/calculations/arithmetic/test_add.py @@ -20,7 +20,11 @@ def test_add_default(fixture_sandbox, aiida_localhost, generate_calc_job): """Test a default `ArithmeticAddCalculation`.""" entry_point_name = 'core.arithmetic.add' - inputs = {'x': orm.Int(1), 'y': orm.Int(2), 'code': orm.Code(remote_computer_exec=(aiida_localhost, '/bin/bash'))} + inputs = { + 'x': orm.Int(1), + 'y': orm.Int(2), + 'code': orm.InstalledCode(computer=aiida_localhost, filepath_executable='/bin/bash') + } calc_info = generate_calc_job(fixture_sandbox, entry_point_name, inputs) options = ArithmeticAddCalculation.spec().inputs['metadata']['options'] @@ -54,7 +58,7 @@ def test_add_custom_filenames(fixture_sandbox, aiida_localhost, generate_calc_jo inputs = { 'x': orm.Int(1), 'y': orm.Int(2), - 'code': orm.Code(remote_computer_exec=(aiida_localhost, '/bin/bash')), + 'code': orm.InstalledCode(computer=aiida_localhost, filepath_executable='/bin/bash'), 'metadata': { 'options': { 'input_filename': input_filename, diff --git a/tests/calculations/test_templatereplacer.py b/tests/calculations/test_templatereplacer.py index 6ed859d20c..3dbaaf23f4 100644 --- a/tests/calculations/test_templatereplacer.py +++ b/tests/calculations/test_templatereplacer.py @@ -24,7 +24,7 @@ def test_base_template(fixture_sandbox, aiida_localhost, generate_calc_job): entry_point_name = 'core.templatereplacer' inputs = { 'code': - orm.Code(remote_computer_exec=(aiida_localhost, '/bin/bash')), + orm.InstalledCode(computer=aiida_localhost, filepath_executable='/bin/bash'), 'metadata': { 'options': { 'resources': { @@ -83,7 +83,7 @@ def test_file_usage(fixture_sandbox, aiida_localhost, generate_calc_job): # Check that the files are correctly copied to the copy list entry_point_name = 'core.templatereplacer' inputs = { - 'code': orm.Code(remote_computer_exec=(aiida_localhost, '/bin/bash')), + 'code': orm.InstalledCode(computer=aiida_localhost, filepath_executable='/bin/bash'), 'metadata': { 'options': { 'resources': { diff --git a/tests/cmdline/commands/test_archive_create.py b/tests/cmdline/commands/test_archive_create.py index ee8fd60205..558646fbf7 100644 --- a/tests/cmdline/commands/test_archive_create.py +++ b/tests/cmdline/commands/test_archive_create.py @@ -14,7 +14,7 @@ import pytest from aiida.cmdline.commands import cmd_archive -from aiida.orm import Code, Computer, Dict, Group +from aiida.orm import Computer, Dict, Group, InstalledCode from aiida.storage.sqlite_zip.migrator import list_versions from aiida.tools.archive import ArchiveFormatSqlZip from tests.utils.archives import get_archive_file @@ -46,7 +46,7 @@ def test_create_all(run_cli_command, tmp_path): scheduler_type='core.direct', workdir='/tmp/aiida' ).store() - code = Code(remote_computer_exec=(computer, '/bin/true')).store() + code = InstalledCode(computer=computer, filepath_executable='/bin/true').store() group = Group(label='test_group').store() filename_output = tmp_path / 'archive.aiida' @@ -56,7 +56,7 @@ def test_create_all(run_cli_command, tmp_path): assert ArchiveFormatSqlZip().read_version(filename_output) == ArchiveFormatSqlZip().latest_version with ArchiveFormatSqlZip().open(filename_output, 'r') as archive: assert archive.querybuilder().append(Computer, project=['uuid']).all(flat=True) == [computer.uuid] - assert archive.querybuilder().append(Code, project=['uuid']).all(flat=True) == [code.uuid] + assert archive.querybuilder().append(InstalledCode, project=['uuid']).all(flat=True) == [code.uuid] assert archive.querybuilder().append(Group, project=['uuid']).all(flat=True) == [group.uuid] @@ -70,7 +70,7 @@ def test_create_basic(run_cli_command, tmp_path): scheduler_type='core.direct', workdir='/tmp/aiida' ).store() - code = Code(remote_computer_exec=(computer, '/bin/true')).store() + code = InstalledCode(computer=computer, filepath_executable='/bin/true').store() group = Group(label='test_group').store() node = Dict().store() filename_output = tmp_path / 'archive.aiida' @@ -81,7 +81,7 @@ def test_create_basic(run_cli_command, tmp_path): assert ArchiveFormatSqlZip().read_version(filename_output) == ArchiveFormatSqlZip().latest_version with ArchiveFormatSqlZip().open(filename_output, 'r') as archive: assert archive.querybuilder().append(Computer, project=['uuid']).all(flat=True) == [computer.uuid] - assert archive.querybuilder().append(Code, project=['uuid']).all(flat=True) == [code.uuid] + assert archive.querybuilder().append(InstalledCode, project=['uuid']).all(flat=True) == [code.uuid] assert archive.querybuilder().append(Group, project=['uuid']).all(flat=True) == [group.uuid] assert archive.querybuilder().append(Dict, project=['uuid']).all(flat=True) == [node.uuid] diff --git a/tests/cmdline/commands/test_calcjob.py b/tests/cmdline/commands/test_calcjob.py index ff619c7ec1..4dc5040f16 100644 --- a/tests/cmdline/commands/test_calcjob.py +++ b/tests/cmdline/commands/test_calcjob.py @@ -38,7 +38,7 @@ def init_profile(self, aiida_profile_clean, aiida_localhost): # pylint: disable # pylint: disable=attribute-defined-outside-init self.computer = aiida_localhost - self.code = orm.Code(remote_computer_exec=(self.computer, '/bin/true')).store() + self.code = orm.InstalledCode(computer=self.computer, filepath_executable='/bin/true').store() self.group = orm.Group(label='test_group').store() self.node = orm.Data().store() self.calcs = [] diff --git a/tests/cmdline/commands/test_code.py b/tests/cmdline/commands/test_code.py index 6ab882cd30..b89a079389 100644 --- a/tests/cmdline/commands/test_code.py +++ b/tests/cmdline/commands/test_code.py @@ -9,9 +9,12 @@ ########################################################################### # pylint: disable=unused-argument,redefined-outer-name """Tests for the 'verdi code' command.""" +import io import os +import pathlib import tempfile import textwrap +import uuid import click import pytest @@ -19,20 +22,22 @@ from aiida.cmdline.commands import cmd_code from aiida.cmdline.params.options.commands.code import validate_label_uniqueness from aiida.common.exceptions import MultipleObjectsError, NotExistent -from aiida.orm import Code, Computer, load_code +from aiida.orm import Computer, InstalledCode, PortableCode, load_code +from aiida.plugins import DataFactory @pytest.fixture def code(aiida_localhost): """Return a ``Code`` instance.""" - code = Code( - input_plugin_name='core.arithmetic.add', - remote_computer_exec=[aiida_localhost, '/remote/abs/path'], + code = InstalledCode( + default_calc_job_plugin='core.arithmetic.add', + computer=aiida_localhost, + filepath_executable='/remote/abs/path', ) code.label = 'code' code.description = 'desc' - code.set_prepend_text('text to prepend') - code.set_append_text('text to append') + code.prepend_text = 'text to prepend' + code.append_text = 'text to append' code.store() return code @@ -53,9 +58,10 @@ def test_code_list_no_codes_error_message(run_cli_command): @pytest.mark.usefixtures('aiida_profile_clean') def test_code_list(run_cli_command, code): """Test ``verdi code list``.""" - code2 = Code( - input_plugin_name='core.templatereplacer', - remote_computer_exec=[code.computer, '/remote/abs/path'], + code2 = InstalledCode( + default_calc_job_plugin='core.templatereplacer', + computer=code.computer, + filepath_executable='/remote/abs/path', ) code2.label = 'code2' code2.store() @@ -71,7 +77,7 @@ def test_code_list(run_cli_command, code): @pytest.mark.usefixtures('aiida_profile_clean') def test_code_list_hide(run_cli_command, code): """Test that hidden codes are shown (or not) properly.""" - code.hide() + code.is_hidden = True options = ['-A'] result = run_cli_command(cmd_code.code_list, options) assert code.full_label not in result.output @@ -85,14 +91,14 @@ def test_code_list_hide(run_cli_command, code): def test_hide_one(run_cli_command, code): """Test ``verdi code hide``.""" run_cli_command(cmd_code.hide, [str(code.pk)]) - assert code.hidden + assert code.is_hidden @pytest.mark.usefixtures('aiida_profile_clean') def test_reveal_one(run_cli_command, code): """Test ``verdi code reveal``.""" run_cli_command(cmd_code.reveal, [str(code.pk)]) - assert not code.hidden + assert not code.is_hidden @pytest.mark.usefixtures('aiida_profile_clean') @@ -102,13 +108,6 @@ def test_relabel_code(run_cli_command, code): assert load_code(code.pk).label == 'new_code' -@pytest.mark.usefixtures('aiida_profile_clean') -def test_relabel_code_full(run_cli_command, code): - """Test ``verdi code relabel`` passing the full code label.""" - run_cli_command(cmd_code.relabel, [str(code.pk), f'new_code@{code.computer.label}']) - assert load_code(code.pk).label == 'new_code' - - @pytest.mark.usefixtures('aiida_profile_clean') def test_relabel_code_full_bad(run_cli_command, code): """Test ``verdi code relabel`` with an incorrect full code label.""" @@ -139,9 +138,9 @@ def test_code_duplicate_non_interactive(run_cli_command, code, non_interactive_e duplicate = load_code(label) assert code.description == duplicate.description - assert code.get_prepend_text() == duplicate.get_prepend_text() - assert code.get_append_text() == duplicate.get_append_text() - assert code.get_input_plugin_name() == duplicate.get_input_plugin_name() + assert code.prepend_text == duplicate.prepend_text + assert code.append_text == duplicate.append_text + assert code.default_calc_job_plugin == duplicate.default_calc_job_plugin @pytest.mark.usefixtures('aiida_profile_clean') @@ -159,7 +158,7 @@ def test_noninteractive_remote(run_cli_command, aiida_localhost, non_interactive '--remote-abs-path=/remote/abs/path', ] run_cli_command(cmd_code.setup_code, options) - assert isinstance(load_code(label), Code) + assert isinstance(load_code(label), InstalledCode) @pytest.mark.usefixtures('aiida_profile_clean') @@ -172,7 +171,7 @@ def test_noninteractive_upload(run_cli_command, non_interactive_editor): '--store-in-db', f'--code-folder={os.path.dirname(__file__)}', f'--code-rel-path={os.path.basename(__file__)}' ] run_cli_command(cmd_code.setup_code, options) - assert isinstance(load_code(label), Code) + assert isinstance(load_code(label), PortableCode) @pytest.mark.usefixtures('aiida_profile_clean') @@ -182,7 +181,7 @@ def test_interactive_remote(run_cli_command, aiida_localhost, non_interactive_ed label = 'interactive_remote' user_input = '\n'.join(['yes', aiida_localhost.label, label, 'desc', 'core.arithmetic.add', '/remote/abs/path']) run_cli_command(cmd_code.setup_code, user_input=user_input) - assert isinstance(load_code(label), Code) + assert isinstance(load_code(label), InstalledCode) @pytest.mark.usefixtures('aiida_profile_clean') @@ -194,7 +193,7 @@ def test_interactive_upload(run_cli_command, non_interactive_editor): basename = os.path.basename(__file__) user_input = '\n'.join(['no', label, 'description', 'core.arithmetic.add', dirname, basename]) run_cli_command(cmd_code.setup_code, user_input=user_input) - assert isinstance(load_code(label), Code) + assert isinstance(load_code(label), PortableCode) @pytest.mark.usefixtures('aiida_profile_clean') @@ -205,7 +204,7 @@ def test_mixed(run_cli_command, aiida_localhost, non_interactive_editor): options = ['--description=description', '--on-computer', '--remote-abs-path=/remote/abs/path'] user_input = '\n'.join([aiida_localhost.label, label, 'core.arithmetic.add']) run_cli_command(cmd_code.setup_code, options, user_input=user_input) - assert isinstance(load_code(label), Code) + assert isinstance(load_code(label), InstalledCode) @pytest.mark.usefixtures('aiida_profile_clean') @@ -219,8 +218,8 @@ def test_code_duplicate_interactive(run_cli_command, aiida_local_code_factory, n duplicate = load_code(label) assert code.description == duplicate.description - assert code.get_prepend_text() == duplicate.get_prepend_text() - assert code.get_append_text() == duplicate.get_append_text() + assert code.prepend_text == duplicate.prepend_text + assert code.append_text == duplicate.append_text @pytest.mark.usefixtures('aiida_profile_clean') @@ -257,7 +256,7 @@ def test_from_config_local_file(non_interactive_editor, run_cli_command, aiida_l handle.write(config_file_template.format(label=label, computer=aiida_localhost.label)) handle.flush() run_cli_command(cmd_code.setup_code, ['--non-interactive', '--config', os.path.realpath(handle.name)]) - assert isinstance(load_code(label), Code) + assert isinstance(load_code(label), InstalledCode) @pytest.mark.usefixtures('aiida_profile_clean') @@ -283,7 +282,7 @@ def test_from_config_url(non_interactive_editor, run_cli_command, aiida_localhos label = 'noninteractive_config_url' fake_url = 'https://my.url.com' run_cli_command(cmd_code.setup_code, ['--non-interactive', '--config', fake_url]) - assert isinstance(load_code(label), Code) + assert isinstance(load_code(label), InstalledCode) @pytest.mark.usefixtures('aiida_profile_clean') @@ -294,12 +293,12 @@ def test_code_setup_remote_duplicate_full_label_interactive( """Test ``verdi code setup`` for a remote code in interactive mode specifying an existing full label.""" label = 'some-label' aiida_local_code_factory('core.arithmetic.add', '/bin/cat', computer=aiida_localhost, label=label) - assert isinstance(load_code(label), Code) + assert isinstance(load_code(label), InstalledCode) label_unique = 'label-unique' user_input = '\n'.join(['yes', aiida_localhost.label, label, label_unique, 'd', 'core.arithmetic.add', '/bin/bash']) run_cli_command(cmd_code.setup_code, user_input=user_input) - assert isinstance(load_code(label_unique), Code) + assert isinstance(load_code(label_unique), InstalledCode) @pytest.mark.usefixtures('aiida_profile_clean') @@ -310,7 +309,7 @@ def test_code_setup_remote_duplicate_full_label_non_interactive( """Test ``verdi code setup`` for a remote code in non-interactive mode specifying an existing full label.""" label = f'some-label-{label_first}' aiida_local_code_factory('core.arithmetic.add', '/bin/cat', computer=aiida_localhost, label=label) - assert isinstance(load_code(label), Code) + assert isinstance(load_code(label), InstalledCode) options = ['-n', '-D', 'd', '-P', 'core.arithmetic.add', '--on-computer', '--remote-abs-path=/remote/abs/path'] @@ -333,15 +332,15 @@ def test_code_setup_local_duplicate_full_label_interactive( filepath.write_text('fake bash') label = 'some-label' - code = Code(local_executable='bash', files=[filepath]) + code = PortableCode(filepath_executable='bash', filepath_files=tmp_path) code.label = label code.store() - assert isinstance(load_code(label), Code) + assert isinstance(load_code(label), PortableCode) label_unique = 'label-unique' user_input = '\n'.join(['no', label, label_unique, 'd', 'core.arithmetic.add', str(tmp_path), filepath.name]) run_cli_command(cmd_code.setup_code, user_input=user_input) - assert isinstance(load_code(label_unique), Code) + assert isinstance(load_code(label_unique), PortableCode) @pytest.mark.usefixtures('aiida_profile_clean') @@ -350,10 +349,11 @@ def test_code_setup_local_duplicate_full_label_non_interactive( ): """Test ``verdi code setup`` for a local code in non-interactive mode specifying an existing full label.""" label = 'some-label' - code = Code(local_executable='bash', files=['/bin/bash']) + code = PortableCode(filepath_executable='bash', filepath_files=pathlib.Path('/bin/bash')) code.label = label + code.base.repository.put_object_from_filelike(io.BytesIO(b''), 'bash') code.store() - assert isinstance(load_code(label), Code) + assert isinstance(load_code(label), PortableCode) options = [ '-n', '-D', 'd', '-P', 'core.arithmetic.add', '--store-in-db', '--code-folder=/bin', '--code-rel-path=bash', @@ -393,7 +393,7 @@ def test_code_test(run_cli_command): computer = Computer( label='test-code-computer', transport_type='core.local', hostname='localhost', scheduler_type='core.slurm' ).store() - code = Code(remote_computer_exec=[computer, '/bin/invalid']).store() + code = InstalledCode(computer=computer, filepath_executable='/bin/invalid').store() result = run_cli_command(cmd_code.code_test, [str(code.pk)], raises=True) assert 'Could not connect to the configured computer' in result.output @@ -401,8 +401,34 @@ def test_code_test(run_cli_command): computer.configure() result = run_cli_command(cmd_code.code_test, [str(code.pk)], raises=True) - assert 'the provided remote absolute path `/bin/invalid` does not exist' in result.output + assert 'The provided remote absolute path `/bin/invalid` does not exist' in result.output - code = Code(remote_computer_exec=[computer, '/bin/bash']).store() + code = InstalledCode(computer=computer, filepath_executable='/bin/bash').store() result = run_cli_command(cmd_code.code_test, [str(code.pk)]) assert 'all tests succeeded.' in result.output + + +@pytest.fixture +def command_options(request, aiida_localhost, tmp_path): + """Return tuple of list of options and entry point.""" + options = [request.param, '-n', '--label', str(uuid.uuid4())] + + if request.param == 'core.code.installed': + options.extend(['--computer', aiida_localhost.pk, '--filepath-executable', '/usr/bin/bash']) + + if request.param == 'core.code.portable': + filepath_executable = 'bash' + (tmp_path / filepath_executable).touch() + options.extend(['--filepath-executable', filepath_executable, '--filepath-files', tmp_path]) + + return options, request.param + + +@pytest.mark.usefixtures('aiida_profile') +@pytest.mark.parametrize('command_options', ('core.code.installed', 'core.code.portable'), indirect=True) +def test_code_create(run_cli_command, command_options): + """Test the ``verdi code create`` command.""" + options, entry_point = command_options + cls = DataFactory(entry_point) + result = run_cli_command(cmd_code.code_create, options) + assert f'Success: Created {cls.__name__}' in result.output diff --git a/tests/cmdline/commands/test_run.py b/tests/cmdline/commands/test_run.py index 6fd599cca1..9ee4b0fc92 100644 --- a/tests/cmdline/commands/test_run.py +++ b/tests/cmdline/commands/test_run.py @@ -166,12 +166,12 @@ def test_no_autogroup(self): @pytest.mark.requires_rmq def test_autogroup_filter_class(self): # pylint: disable=too-many-locals """Check if the autogroup is properly generated but filtered classes are skipped.""" - from aiida.orm import AutoGroup, Code, Node, QueryBuilder, load_node + from aiida.orm import AutoGroup, Node, QueryBuilder, load_node script_content = textwrap.dedent( """\ import sys - from aiida.orm import Computer, Int, ArrayData, KpointsData, CalculationNode, WorkflowNode + from aiida.orm import Computer, Int, ArrayData, KpointsData, CalculationNode, WorkflowNode, InstalledCode from aiida.plugins import CalculationFactory from aiida.engine import run_get_node ArithmeticAdd = CalculationFactory('core.arithmetic.add') @@ -186,9 +186,10 @@ def test_autogroup_filter_class(self): # pylint: disable=too-many-locals ).store() computer.configure() - code = Code( - input_plugin_name='core.arithmetic.add', - remote_computer_exec=[computer, '/bin/true']).store() + code = InstalledCode( + default_calc_job_plugin='core.arithmetic.add', + computer=computer, + filepath_executable='/bin/true').store() inputs = { 'x': Int(1), 'y': Int(2), @@ -218,7 +219,6 @@ def test_autogroup_filter_class(self): # pylint: disable=too-many-locals """ ) - Code() for idx, ( flags, kptdata_in_autogroup, diff --git a/tests/cmdline/params/types/test_code.py b/tests/cmdline/params/types/test_code.py index df55aa888d..74e4497cda 100644 --- a/tests/cmdline/params/types/test_code.py +++ b/tests/cmdline/params/types/test_code.py @@ -13,7 +13,7 @@ import pytest from aiida.cmdline.params.types import CodeParamType -from aiida.orm import Code +from aiida.orm import InstalledCode from aiida.orm.utils.loaders import OrmEntityLoader @@ -31,11 +31,13 @@ def setup_codes(aiida_profile_clean, aiida_localhost): the ID and UUID, respectively, of the first one. This allows us to test the rules implemented to solve ambiguities that arise when determing the identifier type. """ - entity_01 = Code(remote_computer_exec=(aiida_localhost, '/bin/true')).store() - entity_02 = Code(remote_computer_exec=(aiida_localhost, '/bin/true'), - input_plugin_name='core.arithmetic.add').store() - entity_03 = Code(remote_computer_exec=(aiida_localhost, '/bin/true'), - input_plugin_name='core.templatereplacer').store() + entity_01 = InstalledCode(computer=aiida_localhost, filepath_executable='/bin/true').store() + entity_02 = InstalledCode( + computer=aiida_localhost, filepath_executable='/bin/true', default_calc_job_plugin='core.arithmetic.add' + ).store() + entity_03 = InstalledCode( + computer=aiida_localhost, filepath_executable='/bin/true', default_calc_job_plugin='core.templatereplacer' + ).store() entity_01.label = 'computer_01' entity_02.label = str(entity_01.pk) @@ -124,7 +126,7 @@ def test_entry_point_validation(setup_codes): def test_shell_complete(setup_codes, parameter_type, aiida_localhost): """Test the `shell_complete` method that provides auto-complete functionality.""" entity_01, entity_02, entity_03 = setup_codes - entity_04 = Code(label='xavier', remote_computer_exec=(aiida_localhost, '/bin/true')).store() + entity_04 = InstalledCode(label='xavier', computer=aiida_localhost, filepath_executable='/bin/true').store() options = [item.value for item in parameter_type.shell_complete(None, None, '')] assert sorted(options) == sorted([entity_01.label, entity_02.label, entity_03.label, entity_04.label]) diff --git a/tests/conftest.py b/tests/conftest.py index f9550f70f6..e9de501f7a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -373,6 +373,21 @@ def override_logging(isolated_config): configure_logging(with_orm=True) +@pytest.fixture +def suppress_internal_deprecations(): + """Suppress all internal deprecations. + + Warnings emmitted of type :class:`aiida.common.warnings.AiidaDeprecationWarning` for the duration of the test. + """ + import warnings + + from aiida.common.warnings import AiidaDeprecationWarning + + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=AiidaDeprecationWarning) + yield + + @pytest.fixture def with_daemon(): """Starts the daemon process and then makes sure to kill it once the test is done.""" diff --git a/tests/engine/processes/calcjobs/test_calc_job.py b/tests/engine/processes/calcjobs/test_calc_job.py index 1003b6064e..85e96e83ee 100644 --- a/tests/engine/processes/calcjobs/test_calc_job.py +++ b/tests/engine/processes/calcjobs/test_calc_job.py @@ -201,8 +201,8 @@ def test_computer_double_quotes(aiida_local_code_factory, file_regression, compu @pytest.mark.parametrize('code_use_double_quotes', [True, False]) def test_code_double_quotes(aiida_localhost, file_regression, code_use_double_quotes): """test that bash script quote escape behaviour can be controlled""" - code = orm.Code(remote_computer_exec=(aiida_localhost, '/bin/bash')) - code.set_use_double_quotes(code_use_double_quotes) + code = orm.InstalledCode(computer=aiida_localhost, filepath_executable='/bin/bash') + code.use_double_quotes = code_use_double_quotes inputs = { 'code': code.store(), 'metadata': { @@ -261,13 +261,15 @@ class TestCalcJob: """Test for the `CalcJob` process sub class.""" @pytest.fixture(autouse=True) - def init_profile(self, aiida_profile_clean, aiida_localhost): # pylint: disable=unused-argument + def init_profile(self, aiida_profile_clean, aiida_localhost, tmp_path): # pylint: disable=unused-argument """Initialize the profile.""" # pylint: disable=attribute-defined-outside-init + (tmp_path / 'bash').write_bytes(b'bash implementation') + assert Process.current() is None self.computer = aiida_localhost - self.remote_code = orm.Code(remote_computer_exec=(self.computer, '/bin/bash')).store() - self.local_code = orm.Code(local_executable='bash', files=['/bin/bash']).store() + self.remote_code = orm.InstalledCode(computer=self.computer, filepath_executable='/bin/bash').store() + self.local_code = orm.PortableCode(filepath_executable='bash', filepath_files=tmp_path).store() self.inputs = {'x': orm.Int(1), 'y': orm.Int(2), 'metadata': {'options': {}}} yield assert Process.current() is None @@ -434,7 +436,7 @@ def test_par_env_resources_computer(self): computer = orm.Computer('sge_computer', 'localhost', 'desc', 'core.local', 'core.sge').store() computer.set_default_mpiprocs_per_machine(1) - inputs['code'] = orm.Code(remote_computer_exec=(computer, '/bin/bash')).store() + inputs['code'] = orm.InstalledCode(computer=computer, filepath_executable='/bin/bash').store() inputs['metadata']['options']['resources'] = {'parallel_env': 'environment', 'tot_num_mpiprocs': 10} # Just checking that instantiating does not raise, meaning the inputs were valid @@ -493,8 +495,9 @@ def test_rerunnable(self): @pytest.mark.usefixtures('chdir_tmp_path') def test_provenance_exclude_list(self): """Test the functionality of the `CalcInfo.provenance_exclude_list` attribute.""" - code = orm.Code(input_plugin_name='core.arithmetic.add', remote_computer_exec=[self.computer, - '/bin/true']).store() + code = orm.InstalledCode( + default_calc_job_plugin='core.arithmetic.add', computer=self.computer, filepath_executable='/bin/true' + ).store() with tempfile.NamedTemporaryFile('w+') as handle: handle.write('dummy_content') diff --git a/tests/engine/processes/test_builder.py b/tests/engine/processes/test_builder.py index 21d9aa4112..817e255dba 100644 --- a/tests/engine/processes/test_builder.py +++ b/tests/engine/processes/test_builder.py @@ -122,7 +122,7 @@ def test_builder_inputs(): # When no inputs are specified specifically, `prune=True` should get rid of completely empty namespaces assert builder._inputs(prune=False) == {'namespace': {'nested': {}}, 'metadata': {}} - assert builder._inputs(prune=True) == {} + assert not builder._inputs(prune=True) # With a specific input in `namespace` the case of `prune=True` should now only remove `metadata` integer = orm.Int(DEFAULT_INT) @@ -313,10 +313,12 @@ def test_calc_job_node_get_builder_restart(aiida_localhost): def test_code_get_builder(aiida_localhost): """Test that the `Code.get_builder` method returns a builder where the code is already set.""" - code = orm.Code() - code.set_remote_computer_exec((aiida_localhost, '/bin/true')) - code.label = 'test_code' - code.set_input_plugin_name('core.templatereplacer') + code = orm.InstalledCode( + label='test_code', + computer=aiida_localhost, + filepath_executable='/bin/true', + default_calc_job_plugin='core.templatereplacer' + ) code.store() # Check that I can get a builder diff --git a/tests/engine/test_launch.py b/tests/engine/test_launch.py index c187ac31c8..7f7fb89af8 100644 --- a/tests/engine/test_launch.py +++ b/tests/engine/test_launch.py @@ -173,8 +173,9 @@ def test_launchers_dry_run(self): ArithmeticAddCalculation = CalculationFactory('core.arithmetic.add') # pylint: disable=invalid-name - code = orm.Code(input_plugin_name='core.arithmetic.add', remote_computer_exec=[self.computer, - '/bin/true']).store() + code = orm.InstalledCode( + default_calc_job_plugin='core.arithmetic.add', computer=self.computer, filepath_executable='/bin/true' + ).store() inputs = { 'code': code, @@ -214,8 +215,9 @@ def test_launchers_dry_run_no_provenance(self): ArithmeticAddCalculation = CalculationFactory('core.arithmetic.add') # pylint: disable=invalid-name - code = orm.Code(input_plugin_name='core.arithmetic.add', remote_computer_exec=[self.computer, - '/bin/true']).store() + code = orm.InstalledCode( + default_calc_job_plugin='core.arithmetic.add', computer=self.computer, filepath_executable='/bin/true' + ).store() inputs = { 'code': code, @@ -262,8 +264,9 @@ def test_calcjob_dry_run_no_provenance(self): """ import tempfile - code = orm.Code(input_plugin_name='core.arithmetic.add', remote_computer_exec=[self.computer, - '/bin/true']).store() + code = orm.InstalledCode( + default_calc_job_plugin='core.arithmetic.add', computer=self.computer, filepath_executable='/bin/true' + ).store() with tempfile.NamedTemporaryFile('w+') as handle: handle.write('dummy_content') diff --git a/tests/engine/test_process.py b/tests/engine/test_process.py index 01de7bbd75..708456976f 100644 --- a/tests/engine/test_process.py +++ b/tests/engine/test_process.py @@ -238,12 +238,9 @@ def test_valid_cache_hook(self): def test_process_type_with_entry_point(self): """For a process with a registered entry point, the process_type will be its formatted entry point string.""" - from aiida.orm import Code - - code = Code() - code.set_remote_computer_exec((self.computer, '/bin/true')) - code.store() + from aiida.orm import InstalledCode + code = InstalledCode(computer=self.computer, filepath_executable='/bin/true').store() parameters = orm.Dict(dict={}) template = orm.Dict(dict={}) options = { diff --git a/tests/orm/data/code/test_abstract.py b/tests/orm/data/code/test_abstract.py new file mode 100644 index 0000000000..c4976d9067 --- /dev/null +++ b/tests/orm/data/code/test_abstract.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +# pylint: disable=redefined-outer-name +"""Tests for the :class:`aiida.orm.nodes.data.code.abstract.AbstractCode` class.""" +import pytest + +from aiida.orm.nodes.data.code.abstract import AbstractCode + + +class MockCode(AbstractCode): + """Implementation of :class:`aiida.orm.nodes.data.code.abstract.AbstractCode`.""" + + def can_run_on_computer(self, computer) -> bool: + """Return whether the code can run on a given computer.""" + return True + + def get_executable(self) -> str: + """Return the executable that the submission script should execute to run the code.""" + return '' + + @property + def full_label(self) -> str: + """Return the full label of this code.""" + return '' + + +def test_set_label(): + """Test the :meth:`aiida.orm.nodes.data.code.abstract.AbstractCode.label` property setter.""" + label = 'some-label' + code = MockCode(label=label) + assert code.label == label + + code.label = 'alternate-label' + assert code.label == 'alternate-label' + + with pytest.raises(ValueError, match=''): + code.label = 'illegal@label' + + +def test_constructor_defaults(): + """Test the defaults of the constructor.""" + code = MockCode() + assert code.default_calc_job_plugin is None + assert code.append_text == '' + assert code.prepend_text == '' + assert code.use_double_quotes is False + assert code.is_hidden is False diff --git a/tests/orm/data/code/test_installed.py b/tests/orm/data/code/test_installed.py new file mode 100644 index 0000000000..124579325f --- /dev/null +++ b/tests/orm/data/code/test_installed.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +# pylint: disable=redefined-outer-name +"""Tests for the :class:`aiida.orm.nodes.data.code.installed.InstalledCode` class.""" +import pathlib +import uuid + +import pytest + +from aiida.common.exceptions import ModificationNotAllowed, ValidationError +from aiida.orm import Computer +from aiida.orm.nodes.data.code.installed import InstalledCode + + +def test_constructor_raises(aiida_localhost): + """Test the constructor when it is supposed to raise.""" + with pytest.raises(TypeError, match=r'missing .* required positional arguments'): + InstalledCode() # pylint: disable=no-value-for-parameter + + with pytest.raises(TypeError, match=r'Got object of type .*'): + InstalledCode(computer=aiida_localhost, filepath_executable=pathlib.Path('/usr/bin/bash')) + + with pytest.raises(TypeError, match=r'Got object of type .*'): + InstalledCode(computer='computer', filepath_executable='/usr/bin/bash') + + +def test_constructor(aiida_localhost): + """Test the constructor.""" + filepath_executable = '/usr/bin/bash' + code = InstalledCode(computer=aiida_localhost, filepath_executable=filepath_executable) + assert code.computer.pk == aiida_localhost.pk + assert code.filepath_executable == pathlib.PurePath(filepath_executable) + + +def test_validate(aiida_localhost): + """Test the validator is called before storing.""" + filepath_executable = '/usr/bin/bash' + code = InstalledCode(computer=aiida_localhost, filepath_executable=filepath_executable) + + code.computer = None + assert code.computer is None + + with pytest.raises(ValidationError, match='The `computer` is undefined.'): + code.store() + + code.computer = aiida_localhost + code.base.attributes.set(code._KEY_ATTRIBUTE_FILEPATH_EXECUTABLE, None) # pylint: disable=protected-access + + with pytest.raises(ValidationError, match='The `filepath_executable` is not set.'): + code.store() + + code.filepath_executable = filepath_executable + code.store() + assert code.is_stored + + +def test_can_run_on_computer(aiida_localhost): + """Test the :meth:`aiida.orm.nodes.data.code.installed.InstalledCode.can_run_on_computer` method.""" + code = InstalledCode(computer=aiida_localhost, filepath_executable='/usr/bin/bash') + computer = Computer() + + assert code.can_run_on_computer(aiida_localhost) + assert not code.can_run_on_computer(computer) + + +def test_filepath_executable(aiida_localhost): + """Test the :meth:`aiida.orm.nodes.data.code.installed.InstalledCode.filepath_executable` property.""" + filepath_executable = '/usr/bin/bash' + code = InstalledCode(computer=aiida_localhost, filepath_executable=filepath_executable) + assert code.filepath_executable == pathlib.PurePath(filepath_executable) + + filepath_executable = '/usr/bin/cat' + code.filepath_executable = filepath_executable + assert code.filepath_executable == pathlib.PurePath(filepath_executable) + + with pytest.raises(TypeError, match=r'Got object of type .*'): + code.filepath_executable = pathlib.Path(filepath_executable) + + code.store() + + with pytest.raises(ModificationNotAllowed): + code.filepath_executable = filepath_executable + + +def test_validate_filepath_executable(): + """Test the :meth:`aiida.orm.nodes.data.code.installed.InstalledCode.validate_filepath_executable` method.""" + filepath_executable = '/usr/bin/not-existing' + computer = Computer(label=str(uuid.uuid4()), transport_type='core.local') + code = InstalledCode(computer=computer, filepath_executable=filepath_executable) + + with pytest.raises(ValidationError, match=r'Could not connect to the configured computer.*'): + code.validate_filepath_executable() + + computer.configure() + + with pytest.raises(ValidationError, match=r'The provided remote absolute path .* does not exist on the computer\.'): + code.validate_filepath_executable() + + code.filepath_executable = '/usr/bin/bash' + code.validate_filepath_executable() + + +def test_full_label(aiida_localhost): + """Test the :meth:`aiida.orm.nodes.data.code.portable.PortableCode.full_label` property.""" + label = 'some-label' + code = InstalledCode(label=label, computer=aiida_localhost, filepath_executable='/usr/bin/bash') + assert code.full_label == f'{label}@{aiida_localhost.label}' diff --git a/tests/orm/data/code/test_portable.py b/tests/orm/data/code/test_portable.py new file mode 100644 index 0000000000..8c49d06aaa --- /dev/null +++ b/tests/orm/data/code/test_portable.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +# pylint: disable=redefined-outer-name +"""Tests for the :class:`aiida.orm.nodes.data.code.portable.PortableCode` class.""" +import io +import pathlib + +import pytest + +from aiida.common.exceptions import ModificationNotAllowed, ValidationError +from aiida.orm.nodes.data.code.portable import PortableCode + + +def test_constructor_raises(tmp_path): + """Test the constructor when it is supposed to raise.""" + with pytest.raises(TypeError, match=r'missing .* required positional argument'): + PortableCode() # pylint: disable=no-value-for-parameter + + with pytest.raises(TypeError, match=r'Got object of type .*'): + PortableCode(filepath_executable=pathlib.Path('/usr/bin/bash'), filepath_files=tmp_path) + + with pytest.raises(TypeError, match=r'Got object of type .*'): + PortableCode(filepath_executable='bash', filepath_files='string') + + +def test_constructor(tmp_path): + """Test the constructor.""" + (tmp_path / 'bash').touch() + (tmp_path / 'alternate').touch() + filepath_executable = 'bash' + code = PortableCode(filepath_executable=filepath_executable, filepath_files=tmp_path) + assert code.filepath_executable == pathlib.PurePath(filepath_executable) + assert sorted(code.base.repository.list_object_names()) == ['alternate', 'bash'] + + +def test_validate(tmp_path): + """Test the validator is called before storing.""" + filepath_executable = 'bash' + code = PortableCode(filepath_executable=filepath_executable, filepath_files=tmp_path) + + code.base.attributes.set(code._KEY_ATTRIBUTE_FILEPATH_EXECUTABLE, None) # pylint: disable=protected-access + + with pytest.raises(ValidationError, match='The `filepath_executable` is not set.'): + code.store() + + code.filepath_executable = filepath_executable + + with pytest.raises(ValidationError, match=r'The executable .* is not one of the uploaded files'): + code.store() + + code.base.repository.put_object_from_filelike(io.BytesIO(b''), filepath_executable) + code.store() + assert code.is_stored + + +def test_can_run_on_computer(aiida_localhost, tmp_path): + """Test the :meth:`aiida.orm.nodes.data.code.portable.PortableCode.can_run_on_computer` method.""" + code = PortableCode(filepath_executable='./bash', filepath_files=tmp_path) + assert code.can_run_on_computer(aiida_localhost) + + +def test_filepath_executable(tmp_path): + """Test the :meth:`aiida.orm.nodes.data.code.portable.PortableCode.filepath_executable` property.""" + filepath_executable = 'bash' + code = PortableCode(filepath_executable=filepath_executable, filepath_files=tmp_path) + + with pytest.raises(ValueError, match=r'The `filepath_executable` should not be absolute.'): + code.filepath_executable = '/usr/bin/cat' + + with pytest.raises(TypeError, match=r'Got object of type .*'): + code.filepath_executable = pathlib.Path(filepath_executable) + + code.filepath_executable = filepath_executable + code.base.repository.put_object_from_filelike(io.BytesIO(b''), filepath_executable) + code.store() + + with pytest.raises(ModificationNotAllowed): + code.filepath_executable = filepath_executable + + +def test_full_label(tmp_path): + """Test the :meth:`aiida.orm.nodes.data.code.portable.PortableCode.full_label` property.""" + label = 'some-label' + code = PortableCode(label=label, filepath_executable='bash', filepath_files=tmp_path) + assert code.full_label == label diff --git a/tests/orm/data/test_code.py b/tests/orm/data/test_code.py index 6b5864229e..a1907d5ff2 100644 --- a/tests/orm/data/test_code.py +++ b/tests/orm/data/test_code.py @@ -8,20 +8,20 @@ # For further information please visit http://www.aiida.net # ########################################################################### # pylint: disable=redefined-outer-name -"""Tests for :class:`aiida.orm.nodes.data.code.Code` class.""" +"""Tests for :class:`aiida.orm.nodes.data.code.legacy.Code` class.""" import pytest from aiida.common.exceptions import ValidationError from aiida.orm import Code, Computer -@pytest.mark.usefixtures('aiida_profile_clean') +@pytest.mark.usefixtures('aiida_profile_clean', 'suppress_internal_deprecations') def test_validate_remote_exec_path(): """Test ``Code.validate_remote_exec_path``.""" computer = Computer( label='test-code-computer', transport_type='core.local', hostname='localhost', scheduler_type='core.slurm' ).store() - code = Code(remote_computer_exec=[computer, '/bin/invalid']) + code = Code(remote_computer_exec=(computer, '/bin/invalid')) with pytest.raises(ValidationError, match=r'Could not connect to the configured computer.*'): code.validate_remote_exec_path() @@ -31,5 +31,5 @@ def test_validate_remote_exec_path(): with pytest.raises(ValidationError, match=r'the provided remote absolute path `.*` does not exist.*'): code.validate_remote_exec_path() - code = Code(remote_computer_exec=[computer, '/bin/bash']) + code = Code(remote_computer_exec=(computer, '/bin/bash')) code.validate_remote_exec_path() diff --git a/tests/orm/utils/test_loaders.py b/tests/orm/utils/test_loaders.py index 3171926190..9a96ed5a01 100644 --- a/tests/orm/utils/test_loaders.py +++ b/tests/orm/utils/test_loaders.py @@ -52,13 +52,10 @@ def test_load_entity(self): def test_load_code(self): """Test the functionality of load_code.""" - from aiida.orm import Code + from aiida.orm import InstalledCode label = 'compy' - code = Code() - code.label = label - code.set_remote_computer_exec((self.computer, '/x.x')) - code.store() + code = InstalledCode(label=label, computer=self.computer, filepath_executable='/x.x').store() # Load through full label loaded_code = load_code(code.full_label) diff --git a/tests/test_generic.py b/tests/test_generic.py index 024a35b39b..fd9776f52f 100644 --- a/tests/test_generic.py +++ b/tests/test_generic.py @@ -14,8 +14,12 @@ from aiida import orm +@pytest.mark.usefixtures('suppress_internal_deprecations') def test_code_local(aiida_profile_clean, aiida_localhost): - """Test local code.""" + """Test local code. + + Remove this test when legacy `Code` is removed in v3.0. + """ import tempfile from aiida.common.exceptions import ValidationError @@ -37,8 +41,12 @@ def test_code_local(aiida_profile_clean, aiida_localhost): assert code.get_execname(), 'stest.sh' +@pytest.mark.usefixtures('suppress_internal_deprecations') def test_code_remote(aiida_profile_clean, aiida_localhost): - """Test remote code.""" + """Test remote code. + + Remove this test when legacy `Code` is removed in v3.0. + """ import tempfile from aiida.common.exceptions import ValidationError diff --git a/tests/test_nodes.py b/tests/test_nodes.py index 5e959f4c63..48f24e9318 100644 --- a/tests/test_nodes.py +++ b/tests/test_nodes.py @@ -1023,6 +1023,7 @@ def test_comments(self): (default_user_email, 'text2'), ] + @pytest.mark.usefixtures('suppress_internal_deprecations') def test_code_loading_from_string(self): """ Checks that the method Code.get_from_string works correctly. @@ -1070,6 +1071,7 @@ def test_code_loading_from_string(self): with pytest.raises(MultipleObjectsError): orm.Code.get_from_string(code3.label) + @pytest.mark.usefixtures('suppress_internal_deprecations') def test_code_loading_using_get(self): """ Checks that the method Code.get(pk) works correctly. @@ -1129,7 +1131,7 @@ def test_code_loading_using_get(self): pk_label_duplicate = code1.pk code4 = orm.Code() code4.set_remote_computer_exec((self.computer, '/bin/true')) - code4.label = pk_label_duplicate + code4.label = str(pk_label_duplicate) code4.store() # Since the label of code4 is identical to the pk of code1, calling @@ -1140,24 +1142,7 @@ def test_code_loading_using_get(self): assert q_code_4.label == code1.label assert q_code_4.get_remote_exec_path() == code1.get_remote_exec_path() - def test_code_description(self): - """ - This test checks that the code description is retrieved correctly - when the code is searched with its id and label. - """ - # Create a code node - code = orm.Code() - code.set_remote_computer_exec((self.computer, '/bin/true')) - code.label = 'test_code_label' - code.description = 'test code description' - code.store() - - q_code1 = orm.Code.get(label=code.label) - assert code.description == str(q_code1.description) - - q_code2 = orm.Code.get(code.pk) - assert code.description == str(q_code2.description) - + @pytest.mark.usefixtures('suppress_internal_deprecations') def test_list_for_plugin(self): """ This test checks the Code.list_for_plugin() diff --git a/tests/tools/archive/migration/test_prov_redesign.py b/tests/tools/archive/migration/test_prov_redesign.py index 3526f25ca1..f026554392 100644 --- a/tests/tools/archive/migration/test_prov_redesign.py +++ b/tests/tools/archive/migration/test_prov_redesign.py @@ -133,14 +133,13 @@ def test_node_process_type(aiida_profile, tmp_path): assert node.process_type == node_process_type +@pytest.mark.usefixtures('suppress_internal_deprecations') def test_code_type_change(aiida_profile_clean, tmp_path, aiida_localhost): """ Code type string changed Change: “code.Bool.” → “data.code.Code.” """ # Create Code instance - code = orm.Code() - code.set_remote_computer_exec((aiida_localhost, '/bin/true')) - code.store() + code = orm.Code((aiida_localhost, '/bin/true')).store() # Save uuid and type code_uuid = str(code.uuid) diff --git a/tests/tools/archive/orm/test_codes.py b/tests/tools/archive/orm/test_codes.py index a216353348..c93e6d33b0 100644 --- a/tests/tools/archive/orm/test_codes.py +++ b/tests/tools/archive/orm/test_codes.py @@ -21,11 +21,7 @@ def test_that_solo_code_is_exported_correctly(tmp_path, aiida_profile_clean, aii """ code_label = 'test_code1' - code = orm.Code() - code.set_remote_computer_exec((aiida_localhost, '/bin/true')) - code.label = code_label - code.store() - + code = orm.InstalledCode(label=code_label, computer=aiida_localhost, filepath_executable='/bin/true').store() code_uuid = code.uuid export_file = tmp_path / 'export.aiida' @@ -46,11 +42,7 @@ def test_input_code(tmp_path, aiida_profile_clean, aiida_localhost): """ code_label = 'test_code1' - code = orm.Code() - code.set_remote_computer_exec((aiida_localhost, '/bin/true')) - code.label = code_label - code.store() - + code = orm.InstalledCode(label=code_label, computer=aiida_localhost, filepath_executable='/bin/true').store() code_uuid = code.uuid calc = orm.CalcJobNode() @@ -90,11 +82,7 @@ def test_solo_code(tmp_path, aiida_profile_clean, aiida_localhost): """ code_label = 'test_code1' - code = orm.Code() - code.set_remote_computer_exec((aiida_localhost, '/bin/true')) - code.label = code_label - code.store() - + code = orm.InstalledCode(label=code_label, computer=aiida_localhost, filepath_executable='/bin/true').store() code_uuid = code.uuid export_file = tmp_path / 'export.aiida'