diff --git a/CHANGELOG.md b/CHANGELOG.md index 18fcf5174..5385acf5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## Version 0.5.8 (upcoming) + +- add 'mock' module type ## Version 0.5.5 diff --git a/examples/pipelines/mock_pipeline_1.yaml b/examples/pipelines/mock_pipeline_1.yaml new file mode 100644 index 000000000..f36ed40cc --- /dev/null +++ b/examples/pipelines/mock_pipeline_1.yaml @@ -0,0 +1,23 @@ +pipeline_name: mock_pipeline_1 +doc: A pipeline only using the mock module +steps: + - step_id: step_1 + module_type: mock + module_config: + inputs_schema: + first: + type: string + doc: The first string + second: + type: string + doc: The second string + outputs: + combined: + field_schema: + type: string + doc: The combined string + data: "Hello World!" + +input_aliases: + step_1.first: first + step_1.second: second diff --git a/pyproject.toml b/pyproject.toml index f085b66fa..79d7060c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,6 @@ classifiers = [ dependencies = [ - "anyio>=3.7.0", "appdirs>=1.4.4,<2.0.0", "bidict>=0.21.0", "boltons>=21.0.0", @@ -173,6 +172,7 @@ python_api = "kiara:find_model_classes_api" "file_bundle.pick.sub_folder" = "kiara.modules.included_core_modules.filesystem:PickSubBundle" "export.file" = "kiara.modules.included_core_modules.filesystem:ExportFileModule" "render.value" = "kiara.modules.included_core_modules.render_value:ValueTypeRenderModule" +"mock" = "kiara.modules.included_core_modules.mock:MockKiaraModule" [project.entry-points."kiara.operation_types"] diff --git a/src/kiara/models/module/pipeline/__init__.py b/src/kiara/models/module/pipeline/__init__.py index dcc504fd7..8b0fd94e0 100644 --- a/src/kiara/models/module/pipeline/__init__.py +++ b/src/kiara/models/module/pipeline/__init__.py @@ -103,22 +103,49 @@ def create_step( if module_type not in kiara.module_type_names: if module_type in module_map.keys(): + resolved_module_type = module_map[module_type]["module_type"] resolved_module_config = module_map[module_type]["module_config"] + + if module_config: + merged_module_config = dict(resolved_module_config) + merged_module_config.setdefault("defaults", {}) + merged_module_config.setdefault("constants", {}) + defaults = module_config.get("defaults", {}) + constants = module_config.get("constants", {}) + merged_module_config["defaults"].update(defaults) + merged_module_config["constants"].update(constants) + else: + merged_module_config = resolved_module_config + manifest = kiara.create_manifest( module_or_operation=resolved_module_type, - config=resolved_module_config, + config=merged_module_config, ) + elif ( kiara.operation_registry.is_initialized and module_type in kiara.operation_registry.operation_ids ): + op = kiara.operation_registry.operations[module_type] resolved_module_type = op.module_type resolved_module_config = op.module_config + + if module_config: + merged_module_config = dict(resolved_module_config) + merged_module_config.setdefault("defaults", {}) + merged_module_config.setdefault("constants", {}) + defaults = module_config.get("defaults", {}) + constants = module_config.get("constants", {}) + merged_module_config["defaults"].update(defaults) + merged_module_config["constants"].update(constants) + else: + merged_module_config = resolved_module_config + manifest = kiara.create_manifest( module_or_operation=resolved_module_type, - config=resolved_module_config, + config=merged_module_config, ) else: raise InvalidPipelineStepConfig( diff --git a/src/kiara/modules/included_core_modules/mock.py b/src/kiara/modules/included_core_modules/mock.py new file mode 100644 index 000000000..2e651445e --- /dev/null +++ b/src/kiara/modules/included_core_modules/mock.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- + +from typing import Any, ClassVar, Dict, Mapping + +from boltons.strutils import slugify +from pydantic import BaseModel, Field + +from kiara.api import KiaraModule, KiaraModuleConfig, ValueMap, ValueMapSchema +from kiara.defaults import DEFAULT_NO_DESC_VALUE +from kiara.models.module.pipeline import PipelineConfig +from kiara.modules import ModuleCharacteristics + + +class MockOutput(BaseModel): + field_schema: Dict[str, Any] = Field(description="The schema of the output.") + data: Any = Field(description="The data of the output.", default="mock result data") + + +def default_mock_output() -> Dict[str, MockOutput]: + + schema = { + "type": "any", + "doc": "A result", + "optional": False, + } + return {"result": MockOutput(field_schema=schema, data="mock result data")} + + +class MockModuleConfig(KiaraModuleConfig): + + _kiara_model_id: ClassVar = "instance.module_config.mock" + + @classmethod + def create_pipeline_config( + cls, title: str, description: str, author: str, *steps: "MockModuleConfig" + ) -> PipelineConfig: + + data: Dict[str, Any] = { + "pipeline_name": slugify(title), + "doc": description, + "context": {"authors": [author]}, + "steps": [], + } + for step in steps: + step_data = { + "step_id": slugify(step.title), + "module_type": "dummy", + "module_config": { + "title": step.title, + "inputs_schema": step.inputs_schema, + "outputs": step.outputs, + "desc": step.desc, + }, + } + data["steps"].append(step_data) + + pipeline_config = PipelineConfig.from_config(data) + return pipeline_config + + inputs_schema: Dict[str, Dict[str, Any]] = Field( + description="The input fields and their types.", + ) + + outputs: Dict[str, MockOutput] = Field( + description="The outputs fields of the operation, along with their types and mock data.", + default_factory=default_mock_output, + ) + + title: str = Field( + description="The title of this operation.", default="mock_operation" + ) + desc: str = Field( + description="A description of what this step does.", + default=DEFAULT_NO_DESC_VALUE, + ) + + +class MockKiaraModule(KiaraModule): + + _module_type_name = "mock" + _config_cls = MockModuleConfig + + def create_inputs_schema( + self, + ) -> ValueMapSchema: + + result = {} + v: Mapping[str, Any] + for k, v in self.get_config_value("inputs_schema").items(): + data = { + "type": v["type"], + "doc": v.get("doc", "-- n/a --"), + "optional": v.get("optional", True), + } + result[k] = data + + return result + + def create_outputs_schema( + self, + ) -> ValueMapSchema: + + result = {} + field_name: str + field_output: MockOutput + for field_name, field_output in self.get_config_value("outputs").items(): + field_schema = field_output.field_schema + if field_schema: + data = { + "type": field_schema["type"], + "doc": field_schema.get("doc", DEFAULT_NO_DESC_VALUE), + "optional": field_schema.get("optional", False), + } + else: + data = { + "type": "any", + "doc": DEFAULT_NO_DESC_VALUE, + "optional": False, + } + result[field_name] = data + + return result + + def _retrieve_module_characteristics(self) -> ModuleCharacteristics: + + return ModuleCharacteristics( + is_idempotent=True, is_internal=True, unique_result_values=True + ) + + def process(self, inputs: ValueMap, outputs: ValueMap) -> None: + + # config = self.get_config_value("desc") + + mock_outputs = self.get_config_value("outputs") + field_name: str + field_output: MockOutput + for field_name, field_output in mock_outputs.items(): + + outputs.set_value(field_name, field_output.data) diff --git a/tests/test_pipelines/__init__.py b/tests/test_pipelines/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_pipelines/test_pipeline_configs.py b/tests/test_pipelines/test_pipeline_configs.py new file mode 100644 index 000000000..f6c884203 --- /dev/null +++ b/tests/test_pipelines/test_pipeline_configs.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +from kiara.api import KiaraAPI + + +def test_pipeline_default_config_simple(api: KiaraAPI): + + pipeline_config = """ +pipeline_name: test_pipeline +steps: + - step_id: step_1 + module_type: logic.and + - step_id: step_2 + module_type: logic.and + """ + + op = api.get_operation(pipeline_config) + assert op is not None + inputs_schema = op.inputs_schema + outputs_schema = op.outputs_schema + assert len(inputs_schema) == 4 + assert len(outputs_schema) == 2 + + assert inputs_schema["step_1__a"].type == "boolean" + assert outputs_schema["step_1__y"].type == "boolean" + + +def test_pipeline_config_aliases(api: KiaraAPI): + + pipeline_config = """ +pipeline_name: test_pipeline +steps: + - step_id: step_1 + module_type: logic.and + - step_id: step_2 + module_type: logic.and +input_aliases: + step_1.a: a + step_1.b: b + step_2.a: c + step_2.b: d + """ + + op = api.get_operation(pipeline_config) + assert op is not None + inputs_schema = op.inputs_schema + outputs_schema = op.outputs_schema + assert len(inputs_schema) == 4 + assert len(outputs_schema) == 2 + + assert inputs_schema["a"].type == "boolean" + assert inputs_schema["b"].type == "boolean" + assert inputs_schema["c"].type == "boolean" + assert inputs_schema["d"].type == "boolean" + + +def test_pipeline_config_aliases_2(api: KiaraAPI): + + pipeline_config = """ +pipeline_name: test_pipeline +steps: + - step_id: step_1 + module_type: logic.and + - step_id: step_2 + module_type: logic.and +input_aliases: + step_1.a: a + step_1.b: b + step_2.a: a + step_2.b: b + """ + + op = api.get_operation(pipeline_config) + assert op is not None + inputs_schema = op.inputs_schema + outputs_schema = op.outputs_schema + assert len(inputs_schema) == 2 + assert len(outputs_schema) == 2 + + assert inputs_schema["a"].type == "boolean" + assert inputs_schema["b"].type == "boolean" + + +def test_pipeline_module_config(api: KiaraAPI): + + pipeline_config = """ +pipeline_name: test_pipeline +steps: + - step_id: step_1 + module_type: logic.and + module_config: + delay: 0.1 + - step_id: step_2 + module_type: logic.and + module_config: + delay: 0.2 +input_aliases: + step_1.a: a + step_1.b: b + step_2.a: a + step_2.b: b + """ + + api.get_operation(pipeline_config) diff --git a/tests/test_pipelines.py b/tests/test_pipelines/test_pipelines.py similarity index 99% rename from tests/test_pipelines.py rename to tests/test_pipelines/test_pipelines.py index a3ec6915f..9b17211c7 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines/test_pipelines.py @@ -1,12 +1,10 @@ # -*- coding: utf-8 -*- - # Copyright (c) 2021, University of Luxembourg / DHARPA project # Copyright (c) 2021, Markus Binsteiner # # Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/) -# # def test_pipeline_default_controller_invalid_inputs(kiara: Kiara): # # pipeline = kiara.create_pipeline("logic.nand")