test: mock module & tests

Author: makkus

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## Version 0.5.8 (upcoming)
+
+- add 'mock' module type
 
 ## Version 0.5.5
 

examples/pipelines/mock_pipeline_1.yaml

@@ -0,0 +1,23 @@
+pipeline_name: mock_pipeline_1
+doc: A pipeline only using the mock module
+steps:
+  - step_id: step_1
+    module_type: mock
+    module_config:
+      inputs_schema:
+        first:
+          type: string
+          doc: The first string
+        second:
+          type: string
+          doc: The second string
+      outputs:
+        combined:
+          field_schema:
+            type: string
+            doc: The combined string
+          data: "Hello World!"
+
+input_aliases:
+  step_1.first: first
+  step_1.second: second

pyproject.toml

@@ -55,7 +55,6 @@ classifiers = [
 
 
 dependencies = [
-    "anyio>=3.7.0",
     "appdirs>=1.4.4,<2.0.0",
     "bidict>=0.21.0",
     "boltons>=21.0.0",
@@ -173,6 +172,7 @@ python_api = "kiara:find_model_classes_api"
 "file_bundle.pick.sub_folder" = "kiara.modules.included_core_modules.filesystem:PickSubBundle"
 "export.file" = "kiara.modules.included_core_modules.filesystem:ExportFileModule"
 "render.value" = "kiara.modules.included_core_modules.render_value:ValueTypeRenderModule"
+"mock" = "kiara.modules.included_core_modules.mock:MockKiaraModule"
 
 [project.entry-points."kiara.operation_types"]
 

src/kiara/models/module/pipeline/__init__.py

@@ -103,22 +103,49 @@ def create_step(
             if module_type not in kiara.module_type_names:
 
                 if module_type in module_map.keys():
+
                     resolved_module_type = module_map[module_type]["module_type"]
                     resolved_module_config = module_map[module_type]["module_config"]
+
+                    if module_config:
+                        merged_module_config = dict(resolved_module_config)
+                        merged_module_config.setdefault("defaults", {})
+                        merged_module_config.setdefault("constants", {})
+                        defaults = module_config.get("defaults", {})
+                        constants = module_config.get("constants", {})
+                        merged_module_config["defaults"].update(defaults)
+                        merged_module_config["constants"].update(constants)
+                    else:
+                        merged_module_config = resolved_module_config
+
                     manifest = kiara.create_manifest(
                         module_or_operation=resolved_module_type,
-                        config=resolved_module_config,
+                        config=merged_module_config,
                     )
+
                 elif (
                     kiara.operation_registry.is_initialized
                     and module_type in kiara.operation_registry.operation_ids
                 ):
+
                     op = kiara.operation_registry.operations[module_type]
                     resolved_module_type = op.module_type
                     resolved_module_config = op.module_config
+
+                    if module_config:
+                        merged_module_config = dict(resolved_module_config)
+                        merged_module_config.setdefault("defaults", {})
+                        merged_module_config.setdefault("constants", {})
+                        defaults = module_config.get("defaults", {})
+                        constants = module_config.get("constants", {})
+                        merged_module_config["defaults"].update(defaults)
+                        merged_module_config["constants"].update(constants)
+                    else:
+                        merged_module_config = resolved_module_config
+
                     manifest = kiara.create_manifest(
                         module_or_operation=resolved_module_type,
-                        config=resolved_module_config,
+                        config=merged_module_config,
                     )
                 else:
                     raise InvalidPipelineStepConfig(

src/kiara/modules/included_core_modules/mock.py

@@ -0,0 +1,139 @@
+# -*- coding: utf-8 -*-
+
+from typing import Any, ClassVar, Dict, Mapping
+
+from boltons.strutils import slugify
+from pydantic import BaseModel, Field
+
+from kiara.api import KiaraModule, KiaraModuleConfig, ValueMap, ValueMapSchema
+from kiara.defaults import DEFAULT_NO_DESC_VALUE
+from kiara.models.module.pipeline import PipelineConfig
+from kiara.modules import ModuleCharacteristics
+
+
+class MockOutput(BaseModel):
+    field_schema: Dict[str, Any] = Field(description="The schema of the output.")
+    data: Any = Field(description="The data of the output.", default="mock result data")
+
+
+def default_mock_output() -> Dict[str, MockOutput]:
+
+    schema = {
+        "type": "any",
+        "doc": "A result",
+        "optional": False,
+    }
+    return {"result": MockOutput(field_schema=schema, data="mock result data")}
+
+
+class MockModuleConfig(KiaraModuleConfig):
+
+    _kiara_model_id: ClassVar = "instance.module_config.mock"
+
+    @classmethod
+    def create_pipeline_config(
+        cls, title: str, description: str, author: str, *steps: "MockModuleConfig"
+    ) -> PipelineConfig:
+
+        data: Dict[str, Any] = {
+            "pipeline_name": slugify(title),
+            "doc": description,
+            "context": {"authors": [author]},
+            "steps": [],
+        }
+        for step in steps:
+            step_data = {
+                "step_id": slugify(step.title),
+                "module_type": "dummy",
+                "module_config": {
+                    "title": step.title,
+                    "inputs_schema": step.inputs_schema,
+                    "outputs": step.outputs,
+                    "desc": step.desc,
+                },
+            }
+            data["steps"].append(step_data)
+
+        pipeline_config = PipelineConfig.from_config(data)
+        return pipeline_config
+
+    inputs_schema: Dict[str, Dict[str, Any]] = Field(
+        description="The input fields and their types.",
+    )
+
+    outputs: Dict[str, MockOutput] = Field(
+        description="The outputs fields of the operation, along with their types and mock data.",
+        default_factory=default_mock_output,
+    )
+
+    title: str = Field(
+        description="The title of this operation.", default="mock_operation"
+    )
+    desc: str = Field(
+        description="A description of what this step does.",
+        default=DEFAULT_NO_DESC_VALUE,
+    )
+
+
+class MockKiaraModule(KiaraModule):
+
+    _module_type_name = "mock"
+    _config_cls = MockModuleConfig
+
+    def create_inputs_schema(
+        self,
+    ) -> ValueMapSchema:
+
+        result = {}
+        v: Mapping[str, Any]
+        for k, v in self.get_config_value("inputs_schema").items():
+            data = {
+                "type": v["type"],
+                "doc": v.get("doc", "-- n/a --"),
+                "optional": v.get("optional", True),
+            }
+            result[k] = data
+
+        return result
+
+    def create_outputs_schema(
+        self,
+    ) -> ValueMapSchema:
+
+        result = {}
+        field_name: str
+        field_output: MockOutput
+        for field_name, field_output in self.get_config_value("outputs").items():
+            field_schema = field_output.field_schema
+            if field_schema:
+                data = {
+                    "type": field_schema["type"],
+                    "doc": field_schema.get("doc", DEFAULT_NO_DESC_VALUE),
+                    "optional": field_schema.get("optional", False),
+                }
+            else:
+                data = {
+                    "type": "any",
+                    "doc": DEFAULT_NO_DESC_VALUE,
+                    "optional": False,
+                }
+            result[field_name] = data
+
+        return result
+
+    def _retrieve_module_characteristics(self) -> ModuleCharacteristics:
+
+        return ModuleCharacteristics(
+            is_idempotent=True, is_internal=True, unique_result_values=True
+        )
+
+    def process(self, inputs: ValueMap, outputs: ValueMap) -> None:
+
+        # config = self.get_config_value("desc")
+
+        mock_outputs = self.get_config_value("outputs")
+        field_name: str
+        field_output: MockOutput
+        for field_name, field_output in mock_outputs.items():
+
+            outputs.set_value(field_name, field_output.data)

tests/test_pipelines/__init__.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+from kiara.api import KiaraAPI
+
+
+def test_pipeline_default_config_simple(api: KiaraAPI):
+
+    pipeline_config = """
+pipeline_name: test_pipeline
+steps:
+  - step_id: step_1
+    module_type: logic.and
+  - step_id: step_2
+    module_type: logic.and
+     """
+
+    op = api.get_operation(pipeline_config)
+    assert op is not None
+    inputs_schema = op.inputs_schema
+    outputs_schema = op.outputs_schema
+    assert len(inputs_schema) == 4
+    assert len(outputs_schema) == 2
+
+    assert inputs_schema["step_1__a"].type == "boolean"
+    assert outputs_schema["step_1__y"].type == "boolean"
+
+
+def test_pipeline_config_aliases(api: KiaraAPI):
+
+    pipeline_config = """
+pipeline_name: test_pipeline
+steps:
+  - step_id: step_1
+    module_type: logic.and
+  - step_id: step_2
+    module_type: logic.and
+input_aliases:
+  step_1.a: a
+  step_1.b: b
+  step_2.a: c
+  step_2.b: d
+     """
+
+    op = api.get_operation(pipeline_config)
+    assert op is not None
+    inputs_schema = op.inputs_schema
+    outputs_schema = op.outputs_schema
+    assert len(inputs_schema) == 4
+    assert len(outputs_schema) == 2
+
+    assert inputs_schema["a"].type == "boolean"
+    assert inputs_schema["b"].type == "boolean"
+    assert inputs_schema["c"].type == "boolean"
+    assert inputs_schema["d"].type == "boolean"
+
+
+def test_pipeline_config_aliases_2(api: KiaraAPI):
+
+    pipeline_config = """
+pipeline_name: test_pipeline
+steps:
+  - step_id: step_1
+    module_type: logic.and
+  - step_id: step_2
+    module_type: logic.and
+input_aliases:
+  step_1.a: a
+  step_1.b: b
+  step_2.a: a
+  step_2.b: b
+     """
+
+    op = api.get_operation(pipeline_config)
+    assert op is not None
+    inputs_schema = op.inputs_schema
+    outputs_schema = op.outputs_schema
+    assert len(inputs_schema) == 2
+    assert len(outputs_schema) == 2
+
+    assert inputs_schema["a"].type == "boolean"
+    assert inputs_schema["b"].type == "boolean"
+
+
+def test_pipeline_module_config(api: KiaraAPI):
+
+    pipeline_config = """
+pipeline_name: test_pipeline
+steps:
+  - step_id: step_1
+    module_type: logic.and
+    module_config:
+      delay: 0.1
+  - step_id: step_2
+    module_type: logic.and
+    module_config:
+      delay: 0.2
+input_aliases:
+  step_1.a: a
+  step_1.b: b
+  step_2.a: a
+  step_2.b: b
+     """
+
+    api.get_operation(pipeline_config)

tests/test_pipelines/test_pipeline_configs.py

@@ -1,12 +1,10 @@
 # -*- coding: utf-8 -*-
-
 #  Copyright (c) 2021, University of Luxembourg / DHARPA project
 #  Copyright (c) 2021, Markus Binsteiner
 #
 #  Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/)
 
 
-#
 # def test_pipeline_default_controller_invalid_inputs(kiara: Kiara):
 #
 #     pipeline = kiara.create_pipeline("logic.nand")