Merge pull request #234 from scipp/validation

More strict nexus structure validation and tests.

src/beamlime/applications/_nexus_helpers.py

@@ -182,6 +182,30 @@ def _validate_module_keys(
         )
 
 
+def _validate_f144_module_spec(
+    module_spec: StreamModuleValue,
+) -> None:
+    """Validate the f144 module."""
+    if len(module_spec.parent["children"]) != 1:
+        raise InvalidNexusStructureError(
+            "Group containing f144 module should have exactly one child"
+        )
+    if module_spec.dtype is None or module_spec.value_units is None:
+        raise InvalidNexusStructureError(
+            "f144 module spec should have dtype and value_units"
+        )
+
+
+def _validate_ev44_module_spec(
+    module_spec: StreamModuleValue,
+) -> None:
+    """Validate the ev44 module."""
+    if len(module_spec.parent["children"]) != 1:
+        raise InvalidNexusStructureError(
+            "Group containing ev44 module should have exactly one child"
+        )
+
+
 def collect_streaming_modules(
     structure: Mapping,
 ) -> dict[StreamModuleKey, StreamModuleValue]:
@@ -222,7 +246,14 @@ def collect_streaming_modules(
     )
     _validate_module_configs(key_value_pairs)
     _validate_module_keys(key_value_pairs)
-    return dict(key_value_pairs)
+    # Validate each spec
+    for key, value in (key_value_dict := dict(key_value_pairs)).items():
+        if key.module_type == 'ev44':
+            _validate_ev44_module_spec(value)
+        elif key.module_type == 'f144':
+            _validate_f144_module_spec(value)
+
+    return key_value_dict
 
 
 def create_dataset(
@@ -258,8 +289,6 @@ def _is_monitor(group: NexusGroup) -> bool:
 
 def _initialize_ev44(module_spec: StreamModuleValue) -> NexusGroup:
     parent = module_spec.parent
-    if len(parent['children']) != 1:
-        raise ValueError('Group containing ev44 module should have exactly one child')
     group: NexusGroup = cast(NexusGroup, parent.copy())
     group['children'] = [
         create_dataset(
@@ -346,11 +375,6 @@ def _merge_ev44(group: NexusGroup, data: DeserializedMessage) -> None:
 
 def _initialize_f144(module_spec: StreamModuleValue) -> NexusGroup:
     parent = module_spec.parent
-    if len(parent['children']) != 1:
-        raise ValueError('Group containing f144 module should have exactly one child')
-    if module_spec.dtype is None:
-        raise ValueError('f144 module spec should have dtype')
-
     group: NexusGroup = cast(NexusGroup, parent.copy())
     group["children"] = [
         create_dataset(

tests/applications/nexus_helpers_test.py

@@ -1,7 +1,6 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright (c) 2024 Scipp contributors (https://github.com/scipp)
 import hashlib
-import json
 import pathlib
 from collections.abc import Generator, Mapping
 
@@ -112,21 +111,6 @@ def test_find_nexus_structure_not_found_raises() -> None:
         find_nexus_structure({}, ("b0",))
 
 
-def test_invalid_nexus_template_multiple_module_placeholders() -> None:
-    with open(pathlib.Path(__file__).parent / "multiple_modules_datagroup.json") as f:
-        modules = collect_streaming_modules(json.load(f))
-
-    key = StreamModuleKey("ev44", "hypothetical_detector", "ymir_00")
-    spec = modules[key]
-    with pytest.raises(ValueError, match="should have exactly one child"):
-        merge_message_into_nexus_store(
-            module_key=key,
-            module_spec=spec,
-            nexus_store={},
-            data={},  # data does not matter since it reaches the error first.
-        )
-
-
 def test_ymir_detector_template_checksum() -> None:
     """Test that the ymir template with detectors is updated.
 
@@ -343,7 +327,7 @@ def f144_event_generator(shape, dtype):
 
 @pytest.mark.parametrize('shape', [(1,), (2,), (2, 2)])
 @pytest.mark.parametrize('dtype', ['int32', 'uint32', 'float32', 'float64', 'bool'])
-def test_f144(nexus_template_with_streamed_log, shape, dtype):
+def test_f144_merge(nexus_template_with_streamed_log, shape, dtype):
     modules = collect_streaming_modules(nexus_template_with_streamed_log)
     f144_modules = {
         key: value for key, value in modules.items() if key.module_type == 'f144'
@@ -401,7 +385,7 @@ def tdct_event_generator():
         max_last_timestamp = timestamps.max()
 
 
-def test_tdct(nexus_template_with_streamed_tdct: dict):
+def test_tdct_merge(nexus_template_with_streamed_tdct: dict):
     modules = collect_streaming_modules(nexus_template_with_streamed_tdct)
     tdct_modules = {
         key: value for key, value in modules.items() if key.module_type == 'tdct'
@@ -423,15 +407,3 @@ def test_tdct(nexus_template_with_streamed_tdct: dict):
     assert np.issubdtype(
         tdct['config']['values'].dtype, np.dtype(tdct['config']['dtype'])
     )
-
-
-@pytest.fixture()
-def nexus_template_with_mixed_streams(
-    nexus_template_with_streamed_log, nexus_template_with_streamed_tdct
-):
-    return {
-        "children": [
-            nexus_template_with_streamed_log,
-            nexus_template_with_streamed_tdct,
-        ],
-    }

tests/applications/streaming_module_test.py

@@ -1,6 +1,9 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright (c) 2024 Scipp contributors (https://github.com/scipp)
 
+import json
+import pathlib
+
 import pytest
 
 from beamlime.applications._nexus_helpers import (
@@ -28,6 +31,15 @@ def _make_group_with_module_place_holder(
     }
 
 
+def test_invalid_nexus_template_multiple_module_placeholders() -> None:
+    with open(pathlib.Path(__file__).parent / "multiple_modules_datagroup.json") as f:
+        with pytest.raises(
+            InvalidNexusStructureError,
+            match="Group containing ev44 module should have exactly one child",
+        ):
+            collect_streaming_modules(json.load(f))
+
+
 def test_collect_streaming_modules_invalid_missing_topic_raises() -> None:
     invalid_structure = {
         "children": [
@@ -122,11 +134,13 @@ def test_collect_streaming_modules_nxlogs(ymir: dict) -> None:
                             "source": "delay_source_chopper",
                             "topic": "ymir_motion",
                             "dtype": "double",
+                            "value_units": "s",
                         },
                     }
                 ],
             },
             dtype="double",
+            value_units="s",
         ),
         StreamModuleKey(
             "f144", "ymir_motion", "YMIR-SEE:SE-LS336-004:KRDG0"

tests/applications/ymir_detectors.json

@@ -326,7 +326,8 @@
                       "config": {
                         "source": "delay_source_chopper",
                         "topic": "ymir_motion",
-                        "dtype": "double"
+                        "dtype": "double",
+                        "value_units": "s"
                       }
                     }
                   ]
@@ -618,7 +619,8 @@
                       "config": {
                         "source": "YMIR-SETS:SE-BADC-001:SLOWDATA",
                         "topic": "ymir_motion",
-                        "dtype": "double"
+                        "dtype": "double",
+                        "value_units": "mm"
                       }
                     }
                   ]
@@ -639,7 +641,8 @@
                       "config": {
                         "source": "YMIR-SETS:SE-BPTP-001:PTP_STATUS",
                         "topic": "ymir_motion",
-                        "dtype": "double"
+                        "dtype": "double",
+                        "value_units": "mm"
                       }
                     }
                   ]
@@ -1086,7 +1089,8 @@
                       "config": {
                         "source": "YMIR-SpScn:MC-Y-01:Mtr.VELO",
                         "topic": "ymir_motion",
-                        "dtype": "double"
+                        "dtype": "double",
+                        "value_units": "mm"
                       }
                     }
                   ]
@@ -1107,7 +1111,8 @@
                       "config": {
                         "source": "YMIR-SpScn:MC-Z-01:Mtr.VELO",
                         "topic": "ymir_motion",
-                        "dtype": "double"
+                        "dtype": "double",
+                        "value_units": "mm"
                       }
                     }
                   ]
@@ -1128,7 +1133,8 @@
                       "config": {
                         "source": "YMIR-SpScn:MC-X-01:Mtr.VELO",
                         "topic": "ymir_motion",
-                        "dtype": "double"
+                        "dtype": "double",
+                        "value_units": "mm"
                       }
                     }
                   ]
@@ -1184,7 +1190,8 @@
                           "config": {
                             "source": "YMIR-SpScn:MC-Y-01:Mtr-RBV-TSE",
                             "topic": "ymir_motion",
-                            "dtype": "double"
+                            "dtype": "double",
+                            "value_units": "mm"
                           }
                         }
                       ]
@@ -1229,7 +1236,8 @@
                           "config": {
                             "source": "YMIR-SpScn:MC-Z-01:Mtr-RBV-TSE",
                             "topic": "ymir_motion",
-                            "dtype": "double"
+                            "dtype": "double",
+                            "value_units": "mm"
                           }
                         }
                       ]
@@ -1274,7 +1282,8 @@
                           "config": {
                             "source": "YMIR-SpScn:MC-X-01:Mtr-RBV-TSE",
                             "topic": "ymir_motion",
-                            "dtype": "double"
+                            "dtype": "double",
+                            "value_units": "mm"
                           }
                         }
                       ]

tests/helpers/tests/applications/data.py

@@ -19,7 +19,7 @@ def _make_pooch():
             # A version of ymir.json where we added two fake detectors and
             # removed a templating string - to make it like something we might
             # read in a real run start message
-            'ymir_detectors.json': 'md5:02bc6160081a96733c5056cfaa047fca',
+            'ymir_detectors.json': 'md5:d9a25d4375ae3d414d91bfe504baa844',
             'ymir.json': 'md5:5e913075094d97c5e9e9aca76fc32554',
             # readme of the dataset
             'README.md': 'md5:778a0f290894182db5db0170b4f102fa',