From 7bf168ff81de8d6fdff1c87b56a2fb146d52ec31 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Tue, 5 Mar 2024 11:09:18 +0100 Subject: [PATCH 01/25] Depend on scipp, scippnexus --- conda/meta.yaml | 2 ++ pyproject.toml | 2 ++ requirements/base.in | 3 ++- requirements/base.txt | 22 ++++++++++++++++++++-- requirements/basetest.txt | 2 +- requirements/ci.txt | 2 +- requirements/dev.txt | 14 +++++++------- requirements/docs.txt | 15 ++++----------- requirements/mypy.txt | 2 +- requirements/nightly.in | 3 ++- requirements/nightly.txt | 21 ++++++++++++++++++++- requirements/wheels.txt | 2 +- 12 files changed, 63 insertions(+), 27 deletions(-) diff --git a/conda/meta.yaml b/conda/meta.yaml index 4b70d028..96571c25 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -12,6 +12,8 @@ requirements: - setuptools_scm run: - python>=3.10 + - scipp >= 24.02.0 + - scippnexus >= 24.03.0 test: imports: diff --git a/pyproject.toml b/pyproject.toml index 2379e945..92c51469 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,8 @@ requires-python = ">=3.10" # Run 'tox -e deps' after making changes here. This will update requirement files. # Make sure to list one dependency per line. dependencies = [ + "scipp >= 24.02.0", + "scippnexus >= 24.03.0", ] dynamic = ["version"] diff --git a/requirements/base.in b/requirements/base.in index b801db0e..d058df89 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -2,4 +2,5 @@ # will not be touched by ``make_base.py`` # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! - +scipp >= 24.02.0 +scippnexus >= 24.03.0 diff --git a/requirements/base.txt b/requirements/base.txt index c24fd27f..5e1a1dd3 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,8 +1,26 @@ -# SHA1:da39a3ee5e6b4b0d3255bfef95601890afd80709 +# SHA1:b5fdb6600edc83ab95fb0e848607edef52cdd293 # # This file is autogenerated by pip-compile-multi # To update, run: # # pip-compile-multi # - +h5py==3.10.0 + # via scippnexus +numpy==1.26.4 + # via + # h5py + # scipp + # scipy +python-dateutil==2.9.0.post0 + # via scippnexus +scipp==24.2.0 + # via + # -r base.in + # scippnexus +scippnexus==24.3.1 + # via -r base.in +scipy==1.12.0 + # via scippnexus +six==1.16.0 + # via python-dateutil diff --git a/requirements/basetest.txt b/requirements/basetest.txt index bc93620f..a1fc27b6 100644 --- a/requirements/basetest.txt +++ b/requirements/basetest.txt @@ -13,7 +13,7 @@ packaging==23.2 # via pytest pluggy==1.4.0 # via pytest -pytest==8.0.1 +pytest==8.0.2 # via -r basetest.in tomli==2.0.1 # via pytest diff --git a/requirements/ci.txt b/requirements/ci.txt index eeef86ea..87aaf8e4 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -5,7 +5,7 @@ # # pip-compile-multi # -cachetools==5.3.2 +cachetools==5.3.3 # via tox certifi==2024.2.2 # via requests diff --git a/requirements/dev.txt b/requirements/dev.txt index 61cfd1bb..42476c74 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -50,7 +50,7 @@ isoduration==20.11.0 # via jsonschema jinja2-ansible-filters==1.3.2 # via copier -json5==0.9.17 +json5==0.9.20 # via jupyterlab-server jsonpointer==2.4 # via jsonschema @@ -61,9 +61,9 @@ jsonschema[format-nongpl]==4.21.1 # nbformat jupyter-events==0.9.0 # via jupyter-server -jupyter-lsp==2.2.2 +jupyter-lsp==2.2.3 # via jupyterlab -jupyter-server==2.12.5 +jupyter-server==2.13.0 # via # jupyter-lsp # jupyterlab @@ -71,7 +71,7 @@ jupyter-server==2.12.5 # notebook-shim jupyter-server-terminals==0.5.2 # via jupyter-server -jupyterlab==4.1.2 +jupyterlab==4.1.3 # via -r dev.in jupyterlab-server==2.25.3 # via jupyterlab @@ -91,9 +91,9 @@ prometheus-client==0.20.0 # via jupyter-server pycparser==2.21 # via cffi -pydantic==2.6.1 +pydantic==2.6.3 # via copier -pydantic-core==2.16.2 +pydantic-core==2.16.3 # via pydantic python-json-logger==2.0.7 # via jupyter-events @@ -111,7 +111,7 @@ rfc3986-validator==0.1.1 # jupyter-events send2trash==1.8.2 # via jupyter-server -sniffio==1.3.0 +sniffio==1.3.1 # via # anyio # httpx diff --git a/requirements/docs.txt b/requirements/docs.txt index caed3661..4381e0a2 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -54,9 +54,9 @@ idna==3.6 # via requests imagesize==1.4.1 # via sphinx -ipykernel==6.29.2 +ipykernel==6.29.3 # via -r docs.in -ipython==8.22.0 +ipython==8.22.2 # via # -r docs.in # ipykernel @@ -107,7 +107,7 @@ myst-parser==2.0.0 # via -r docs.in nbclient==0.9.0 # via nbconvert -nbconvert==7.16.1 +nbconvert==7.16.2 # via nbsphinx nbformat==5.9.2 # via @@ -149,8 +149,6 @@ pygments==2.17.2 # nbconvert # pydata-sphinx-theme # sphinx -python-dateutil==2.8.2 - # via jupyter-client pyyaml==6.0.1 # via myst-parser pyzmq==25.1.2 @@ -167,11 +165,6 @@ rpds-py==0.18.0 # via # jsonschema # referencing -six==1.16.0 - # via - # asttokens - # bleach - # python-dateutil snowballstemmer==2.2.0 # via sphinx soupsieve==2.5 @@ -223,7 +216,7 @@ traitlets==5.14.1 # nbconvert # nbformat # nbsphinx -typing-extensions==4.9.0 +typing-extensions==4.10.0 # via pydata-sphinx-theme urllib3==2.2.1 # via requests diff --git a/requirements/mypy.txt b/requirements/mypy.txt index ac285686..49722576 100644 --- a/requirements/mypy.txt +++ b/requirements/mypy.txt @@ -10,5 +10,5 @@ mypy==1.8.0 # via -r mypy.in mypy-extensions==1.0.0 # via mypy -typing-extensions==4.9.0 +typing-extensions==4.10.0 # via mypy diff --git a/requirements/nightly.in b/requirements/nightly.in index 6b1ebcc2..0e1f5905 100644 --- a/requirements/nightly.in +++ b/requirements/nightly.in @@ -1,4 +1,5 @@ -r basetest.in # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! - +scipp >= 24.02.0 +scippnexus >= 24.03.0 diff --git a/requirements/nightly.txt b/requirements/nightly.txt index a98564b2..2126652e 100644 --- a/requirements/nightly.txt +++ b/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:e8b11c1210855f07eaedfbcfb3ecd1aec3595dee +# SHA1:9bb7ade09fe2af7ab62c586f5f5dc6f3e9b8344b # # This file is autogenerated by pip-compile-multi # To update, run: @@ -6,3 +6,22 @@ # pip-compile-multi # -r basetest.txt +h5py==3.10.0 + # via scippnexus +numpy==1.26.4 + # via + # h5py + # scipp + # scipy +python-dateutil==2.9.0.post0 + # via scippnexus +scipp==24.2.0 + # via + # -r nightly.in + # scippnexus +scippnexus==24.3.1 + # via -r nightly.in +scipy==1.12.0 + # via scippnexus +six==1.16.0 + # via python-dateutil diff --git a/requirements/wheels.txt b/requirements/wheels.txt index 2e33cfa3..12ac3232 100644 --- a/requirements/wheels.txt +++ b/requirements/wheels.txt @@ -5,7 +5,7 @@ # # pip-compile-multi # -build==1.0.3 +build==1.1.1 # via -r wheels.in packaging==23.2 # via build From 28e1f60b87fad43a102bee2fcf9f9a16891f87c8 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Tue, 5 Mar 2024 11:32:24 +0100 Subject: [PATCH 02/25] Skeleton for load_detector --- src/ess/reduce/__init__.py | 4 +++ src/ess/reduce/nexus.py | 52 ++++++++++++++++++++++++++++++++++++++ tests/nexus_test.py | 52 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+) create mode 100644 src/ess/reduce/nexus.py create mode 100644 tests/nexus_test.py diff --git a/src/ess/reduce/__init__.py b/src/ess/reduce/__init__.py index 9da9479a..3fa47da5 100644 --- a/src/ess/reduce/__init__.py +++ b/src/ess/reduce/__init__.py @@ -4,9 +4,13 @@ # flake8: noqa import importlib.metadata +from . import nexus + try: __version__ = importlib.metadata.version(__package__ or __name__) except importlib.metadata.PackageNotFoundError: __version__ = "0.0.0" del importlib + +__all__ = ['nexus'] diff --git a/src/ess/reduce/nexus.py b/src/ess/reduce/nexus.py new file mode 100644 index 00000000..5e1ed8bd --- /dev/null +++ b/src/ess/reduce/nexus.py @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) + +from contextlib import nullcontext +from pathlib import Path +from typing import BinaryIO, ContextManager, NewType, Optional, Union + +import scipp as sc +import scippnexus as snx + +FilePath = NewType('FilePath', Path) +"""Full path to a NeXus file on disk.""" +NeXusFile = NewType('NeXusFile', BinaryIO) +"""An open NeXus file. + +Can be any file handle for reading binary data. + +Note that this cannot be used as a parameter in Sciline as there are no +concrete implementations of ``BinaryIO``. +The type alias is provided for callers of load functions outside of pipelines. +""" +NeXusGroup = NewType('NeXusGroup', snx.Group) +"""A ScippNexus group in an open file.""" + +InstrumentName = NewType('InstrumentName', str) +"""Name of an instrument in a NeXus file.""" +DetectorName = NewType('DetectorName', str) +"""Name of a detector (bank) in a NeXus file.""" + +RawDetector = NewType('RawDetector', sc.DataArray) +"""A Scipp DataArray containing raw data from a detector.""" +RawMonitor = NewType('RawMonitor', sc.DataArray) +"""A Scipp DataArray containing raw data from a monitor.""" + + +def load_detector( + file_path: Union[FilePath, NeXusFile, NeXusGroup], + instrument_name: Optional[InstrumentName] = None, + detector_name: Optional[DetectorName] = None, +) -> RawDetector: + with _open_nexus_file(file_path) as f: + return RawDetector( + f[f'entry/{instrument_name}/{detector_name}/{detector_name}_events'][...] + ) + + +def _open_nexus_file( + file_path: Union[FilePath, NeXusFile, NeXusGroup] +) -> ContextManager: + if isinstance(file_path, NeXusGroup.__supertype__): + return nullcontext(file_path) + return snx.File(file_path) diff --git a/tests/nexus_test.py b/tests/nexus_test.py new file mode 100644 index 00000000..a6fbe5f2 --- /dev/null +++ b/tests/nexus_test.py @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) + +import h5py +import pytest +import scipp as sc +import scipp.testing +import scippnexus as snx + +from ess.reduce import nexus + + +@pytest.fixture() +def nxroot(): + """Yield NXroot containing a single NXentry named 'entry'""" + with h5py.File('dummy.nxs', mode='w', driver="core", backing_store=False) as f: + root = snx.Group(f, definitions=snx.base_definitions()) + root.create_class('entry', snx.NXentry) + yield root + + +@pytest.fixture() +def nexus_group(nxroot): + instrument = nxroot['entry'].create_class('reducer', snx.NXinstrument) + detector = instrument.create_class('bank12', snx.NXdetector) + + events = detector.create_class('bank12_events', snx.NXevent_data) + events['event_id'] = sc.array(dims=[''], unit=None, values=[1, 2, 4, 1, 2, 2]) + events['event_time_offset'] = sc.array( + dims=[''], unit='s', values=[456, 7, 3, 345, 632, 23] + ) + events['event_time_zero'] = sc.array(dims=[''], unit='s', values=[1, 2, 3, 4]) + events['event_index'] = sc.array(dims=[''], unit=None, values=[0, 3, 3, -1000]) + + return nxroot + + +# TODO histogram data +# TODO test with real file + BytesIO + snx.Group + + +@pytest.mark.parametrize('detector_name', (None, nexus.DetectorName('bank12'))) +@pytest.mark.parametrize('instrument_name', (None, nexus.InstrumentName('reducer'))) +def test_load_detector_from_group(nexus_group, instrument_name, detector_name): + detector = nexus.load_detector( + nexus.NeXusGroup(nexus_group), + instrument_name=instrument_name, + detector_name=detector_name, + ) + expected = nexus_group['entry/reducer/bank12/bank12_events'][...] + # TODO positions + sc.testing.assert_identical(detector, expected) From aa43050ae65f4074597ed97d9f3c9e08f36e4a66 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Tue, 5 Mar 2024 11:38:26 +0100 Subject: [PATCH 03/25] Deduce instrument and detector --- src/ess/reduce/nexus.py | 48 +++++++++++++++++++++++++++++++++++++---- tests/nexus_test.py | 34 +++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/src/ess/reduce/nexus.py b/src/ess/reduce/nexus.py index 5e1ed8bd..e1a6f31a 100644 --- a/src/ess/reduce/nexus.py +++ b/src/ess/reduce/nexus.py @@ -1,9 +1,10 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +import warnings from contextlib import nullcontext from pathlib import Path -from typing import BinaryIO, ContextManager, NewType, Optional, Union +from typing import BinaryIO, ContextManager, NewType, Optional, Type, Union import scipp as sc import scippnexus as snx @@ -38,15 +39,54 @@ def load_detector( instrument_name: Optional[InstrumentName] = None, detector_name: Optional[DetectorName] = None, ) -> RawDetector: + """ + TODO handling of names, including event name + """ with _open_nexus_file(file_path) as f: - return RawDetector( - f[f'entry/{instrument_name}/{detector_name}/{detector_name}_events'][...] + entry = f['entry'] + instrument = _unique_child_group(entry, snx.NXinstrument, instrument_name) + detector = _unique_child_group(instrument, snx.NXdetector, detector_name) + events = _unique_child_group( + detector, + snx.NXevent_data, + None if detector_name is None else f'{detector_name}_events', ) + data = events[()] + if not isinstance(data, sc.DataArray): + warnings.warn( + 'NeXus (event)data was not assembled correctly. Expected a ' + f'scipp.DataArray, but got {type(events)}.', + UserWarning, + stacklevel=2, + ) + return RawDetector(data) # type: ignore[arg-type] def _open_nexus_file( file_path: Union[FilePath, NeXusFile, NeXusGroup] ) -> ContextManager: - if isinstance(file_path, NeXusGroup.__supertype__): + if isinstance(file_path, getattr(NeXusGroup, '__supertype__', type(None))): return nullcontext(file_path) return snx.File(file_path) + + +def _unique_child_group( + group: snx.Group, nx_class: Type[snx.NXobject], name: Optional[str] +) -> snx.Group: + if name is not None: + child = group[name] + if isinstance(child, snx.Field): + raise ValueError( + f"Expected a NeXus group as item '{name}' but got a field." + ) + if child.nx_class != nx_class: + raise ValueError( + f'The NeXus group {name} was expected to be a ' + f'{nx_class} but is a {child.nx_class}.' + ) + return child + + children = group[nx_class] + if len(children) != 1: + raise ValueError(f'Expected exactly one {nx_class} group.') + return next(iter(children.values())) # type: ignore[return-value] diff --git a/tests/nexus_test.py b/tests/nexus_test.py index a6fbe5f2..267db6d8 100644 --- a/tests/nexus_test.py +++ b/tests/nexus_test.py @@ -35,6 +35,16 @@ def nexus_group(nxroot): return nxroot +@pytest.fixture() +def nexus_group_two_banks(nexus_group): + # Make a second bank + bank13 = nexus_group['entry/reducer'].create_class('bank13', snx.NXdetector) + events = bank13.create_class('bank13_events', snx.NXevent_data) + for key, val in nexus_group['entry/reducer/bank12/bank12_events'].items(): + events[key] = val[()] + return nexus_group + + # TODO histogram data # TODO test with real file + BytesIO + snx.Group @@ -50,3 +60,27 @@ def test_load_detector_from_group(nexus_group, instrument_name, detector_name): expected = nexus_group['entry/reducer/bank12/bank12_events'][...] # TODO positions sc.testing.assert_identical(detector, expected) + + +@pytest.mark.parametrize('instrument_name', (None, nexus.InstrumentName('reducer'))) +def test_load_detector_needs_unique_detector_if_no_name_given( + nexus_group_two_banks, instrument_name +): + with pytest.raises(ValueError): + nexus.load_detector( + nexus.NeXusGroup(nexus_group_two_banks), + instrument_name=instrument_name, + detector_name=None, + ) + + +@pytest.mark.parametrize('instrument_name', (None, nexus.InstrumentName('reducer'))) +def test_load_detector_select_bank(nexus_group_two_banks, instrument_name): + detector = nexus.load_detector( + nexus.NeXusGroup(nexus_group_two_banks), + instrument_name=instrument_name, + detector_name=nexus.DetectorName('bank13'), + ) + expected = nexus_group_two_banks['entry/reducer/bank13/bank13_events'][...] + # TODO positions + sc.testing.assert_identical(detector, expected) From d57451a0fa946232889b459f2dcef7ffba01acac Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Tue, 5 Mar 2024 13:49:44 +0100 Subject: [PATCH 04/25] use pyfakefs --- conda/meta.yaml | 1 + requirements/basetest.in | 1 + requirements/basetest.txt | 4 +++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/conda/meta.yaml b/conda/meta.yaml index 96571c25..745b286b 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -19,6 +19,7 @@ test: imports: - ess.reduce requires: + - pyfakefs - pytest source_files: - pyproject.toml diff --git a/requirements/basetest.in b/requirements/basetest.in index e4a48b29..a97ea118 100644 --- a/requirements/basetest.in +++ b/requirements/basetest.in @@ -1,4 +1,5 @@ # Dependencies that are only used by tests. # Do not make an environment from this file, use test.txt instead! +pyfakefs pytest diff --git a/requirements/basetest.txt b/requirements/basetest.txt index a1fc27b6..9b28aa74 100644 --- a/requirements/basetest.txt +++ b/requirements/basetest.txt @@ -1,4 +1,4 @@ -# SHA1:0eaa389e1fdb3a1917c0f987514bd561be5718ee +# SHA1:81a76547a01dccca58b7135a0696f0d92bc197c3 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -13,6 +13,8 @@ packaging==23.2 # via pytest pluggy==1.4.0 # via pytest +pyfakefs==5.3.5 + # via -r basetest.in pytest==8.0.2 # via -r basetest.in tomli==2.0.1 From 2c8c951576a4943f88b7c16c3f74cf7d437c7533 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Tue, 5 Mar 2024 13:50:02 +0100 Subject: [PATCH 05/25] Test with file path and buffer --- src/ess/reduce/nexus.py | 9 +-- tests/nexus_test.py | 126 ++++++++++++++++++++++------------------ 2 files changed, 75 insertions(+), 60 deletions(-) diff --git a/src/ess/reduce/nexus.py b/src/ess/reduce/nexus.py index e1a6f31a..ffece202 100644 --- a/src/ess/reduce/nexus.py +++ b/src/ess/reduce/nexus.py @@ -36,8 +36,9 @@ def load_detector( file_path: Union[FilePath, NeXusFile, NeXusGroup], + *, + detector_name: DetectorName, instrument_name: Optional[InstrumentName] = None, - detector_name: Optional[DetectorName] = None, ) -> RawDetector: """ TODO handling of names, including event name @@ -49,7 +50,7 @@ def load_detector( events = _unique_child_group( detector, snx.NXevent_data, - None if detector_name is None else f'{detector_name}_events', + f'{detector_name}_events', ) data = events[()] if not isinstance(data, sc.DataArray): @@ -81,12 +82,12 @@ def _unique_child_group( ) if child.nx_class != nx_class: raise ValueError( - f'The NeXus group {name} was expected to be a ' + f"The NeXus group '{name}' was expected to be a " f'{nx_class} but is a {child.nx_class}.' ) return child children = group[nx_class] if len(children) != 1: - raise ValueError(f'Expected exactly one {nx_class} group.') + raise ValueError(f'Expected exactly one {nx_class} group, got {len(children)}') return next(iter(children.values())) # type: ignore[return-value] diff --git a/tests/nexus_test.py b/tests/nexus_test.py index 267db6d8..2e633138 100644 --- a/tests/nexus_test.py +++ b/tests/nexus_test.py @@ -1,7 +1,11 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) -import h5py +import uuid +from io import BytesIO +from pathlib import Path +from typing import Union + import pytest import scipp as sc import scipp.testing @@ -10,77 +14,87 @@ from ess.reduce import nexus -@pytest.fixture() -def nxroot(): - """Yield NXroot containing a single NXentry named 'entry'""" - with h5py.File('dummy.nxs', mode='w', driver="core", backing_store=False) as f: - root = snx.Group(f, definitions=snx.base_definitions()) - root.create_class('entry', snx.NXentry) - yield root +def _event_data_components() -> sc.DataGroup: + return sc.DataGroup( + { + 'event_id': sc.array(dims=['event'], unit=None, values=[1, 2, 4, 1, 2, 2]), + 'event_time_offset': sc.array( + dims=['event'], unit='s', values=[456, 7, 3, 345, 632, 23] + ), + 'event_time_zero': sc.array( + dims=['event_time_zero'], unit='s', values=[1, 2, 3, 4] + ), + 'event_index': sc.array( + dims=['event_time_zero'], unit=None, values=[0, 3, 3, 6] + ), + } + ) -@pytest.fixture() -def nexus_group(nxroot): - instrument = nxroot['entry'].create_class('reducer', snx.NXinstrument) - detector = instrument.create_class('bank12', snx.NXdetector) - - events = detector.create_class('bank12_events', snx.NXevent_data) - events['event_id'] = sc.array(dims=[''], unit=None, values=[1, 2, 4, 1, 2, 2]) - events['event_time_offset'] = sc.array( - dims=[''], unit='s', values=[456, 7, 3, 345, 632, 23] - ) - events['event_time_zero'] = sc.array(dims=[''], unit='s', values=[1, 2, 3, 4]) - events['event_index'] = sc.array(dims=[''], unit=None, values=[0, 3, 3, -1000]) +def _write_nexus_events(store: Union[Path, BytesIO]) -> None: + with snx.File(store, 'w') as root: + entry = root.create_class('entry', snx.NXentry) + instrument = entry.create_class('reducer', snx.NXinstrument) + detector = instrument.create_class('bank12', snx.NXdetector) - return nxroot + events = detector.create_class('bank12_events', snx.NXevent_data) + for key, val in _event_data_components().items(): + events[key] = val -@pytest.fixture() -def nexus_group_two_banks(nexus_group): - # Make a second bank - bank13 = nexus_group['entry/reducer'].create_class('bank13', snx.NXdetector) - events = bank13.create_class('bank13_events', snx.NXevent_data) - for key, val in nexus_group['entry/reducer/bank12/bank12_events'].items(): - events[key] = val[()] - return nexus_group +def _file_store(typ): + if typ == Path: + return Path('nexus_test', uuid.uuid4().hex, 'testfile.nxs') + return BytesIO() -# TODO histogram data -# TODO test with real file + BytesIO + snx.Group +@pytest.fixture(params=[Path, BytesIO, snx.Group]) +def nexus_file(fs, request): + store = _file_store(request.param) + _write_nexus_events(store) + if isinstance(store, BytesIO): + store.seek(0) + if request.param in (Path, BytesIO): + yield store + else: + with snx.File(store, 'r') as f: + yield f -@pytest.mark.parametrize('detector_name', (None, nexus.DetectorName('bank12'))) -@pytest.mark.parametrize('instrument_name', (None, nexus.InstrumentName('reducer'))) -def test_load_detector_from_group(nexus_group, instrument_name, detector_name): - detector = nexus.load_detector( - nexus.NeXusGroup(nexus_group), - instrument_name=instrument_name, - detector_name=detector_name, + +@pytest.fixture() +def expected_bank12(): + components = _event_data_components() + buffer = sc.DataArray( + sc.ones(sizes={'event': 6}, unit='counts'), + coords={ + 'event_id': components['event_id'], + 'event_time_offset': components['event_time_offset'], + }, + ) + events = sc.bins( + data=buffer, + begin=components['event_index'], + end=sc.concat( + [components['event_index'][1:], components['event_index'][-1]], + dim='event_time_zero', + ), + dim='event', + ) + return sc.DataArray( + events, coords={'event_time_zero': components['event_time_zero']} ) - expected = nexus_group['entry/reducer/bank12/bank12_events'][...] - # TODO positions - sc.testing.assert_identical(detector, expected) -@pytest.mark.parametrize('instrument_name', (None, nexus.InstrumentName('reducer'))) -def test_load_detector_needs_unique_detector_if_no_name_given( - nexus_group_two_banks, instrument_name -): - with pytest.raises(ValueError): - nexus.load_detector( - nexus.NeXusGroup(nexus_group_two_banks), - instrument_name=instrument_name, - detector_name=None, - ) +# TODO histogram data @pytest.mark.parametrize('instrument_name', (None, nexus.InstrumentName('reducer'))) -def test_load_detector_select_bank(nexus_group_two_banks, instrument_name): +def test_load_detector(nexus_file, expected_bank12, instrument_name): detector = nexus.load_detector( - nexus.NeXusGroup(nexus_group_two_banks), + nexus.NeXusGroup(nexus_file), + detector_name=nexus.DetectorName('bank12'), instrument_name=instrument_name, - detector_name=nexus.DetectorName('bank13'), ) - expected = nexus_group_two_banks['entry/reducer/bank13/bank13_events'][...] # TODO positions - sc.testing.assert_identical(detector, expected) + sc.testing.assert_identical(detector, nexus.RawDetector(expected_bank12)) From 79804d1dd25ce61b336376000b53e0c701edab1a Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Tue, 5 Mar 2024 13:53:47 +0100 Subject: [PATCH 06/25] Wrap RawDetector in a data group --- src/ess/reduce/nexus.py | 8 ++++---- tests/nexus_test.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ess/reduce/nexus.py b/src/ess/reduce/nexus.py index ffece202..844d2db4 100644 --- a/src/ess/reduce/nexus.py +++ b/src/ess/reduce/nexus.py @@ -28,10 +28,10 @@ DetectorName = NewType('DetectorName', str) """Name of a detector (bank) in a NeXus file.""" -RawDetector = NewType('RawDetector', sc.DataArray) -"""A Scipp DataArray containing raw data from a detector.""" +RawDetector = NewType('RawDetector', sc.DataGroup) +"""Raw data from a NeXus detector.""" RawMonitor = NewType('RawMonitor', sc.DataArray) -"""A Scipp DataArray containing raw data from a monitor.""" +"""Raw data from a NeXus monitor.""" def load_detector( @@ -60,7 +60,7 @@ def load_detector( UserWarning, stacklevel=2, ) - return RawDetector(data) # type: ignore[arg-type] + return RawDetector(sc.DataGroup(data=data)) def _open_nexus_file( diff --git a/tests/nexus_test.py b/tests/nexus_test.py index 2e633138..b690c1cb 100644 --- a/tests/nexus_test.py +++ b/tests/nexus_test.py @@ -97,4 +97,4 @@ def test_load_detector(nexus_file, expected_bank12, instrument_name): instrument_name=instrument_name, ) # TODO positions - sc.testing.assert_identical(detector, nexus.RawDetector(expected_bank12)) + sc.testing.assert_identical(detector['data'], expected_bank12) From 3c20d3d321498a3b5aaaa3a2ebeee58d1df663f7 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Tue, 5 Mar 2024 15:34:50 +0100 Subject: [PATCH 07/25] Add load_monitor and support dense data --- src/ess/reduce/nexus.py | 58 ++++++++++++++++++++++++++++------------- tests/nexus_test.py | 55 +++++++++++++++++++++++++++++++++----- 2 files changed, 89 insertions(+), 24 deletions(-) diff --git a/src/ess/reduce/nexus.py b/src/ess/reduce/nexus.py index 844d2db4..c77480f7 100644 --- a/src/ess/reduce/nexus.py +++ b/src/ess/reduce/nexus.py @@ -1,10 +1,9 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) -import warnings from contextlib import nullcontext from pathlib import Path -from typing import BinaryIO, ContextManager, NewType, Optional, Type, Union +from typing import BinaryIO, ContextManager, NewType, Optional, Type, Union, cast import scipp as sc import scippnexus as snx @@ -27,10 +26,12 @@ """Name of an instrument in a NeXus file.""" DetectorName = NewType('DetectorName', str) """Name of a detector (bank) in a NeXus file.""" +MonitorName = NewType('MonitorName', str) +"""Name of a monitor in a NeXus file.""" RawDetector = NewType('RawDetector', sc.DataGroup) """Raw data from a NeXus detector.""" -RawMonitor = NewType('RawMonitor', sc.DataArray) +RawMonitor = NewType('RawMonitor', sc.DataGroup) """Raw data from a NeXus monitor.""" @@ -43,24 +44,45 @@ def load_detector( """ TODO handling of names, including event name """ + return RawDetector( + _load_data( + file_path, + detector_name=detector_name, + detector_class=snx.NXdetector, + instrument_name=instrument_name, + ) + ) + + +def load_monitor( + file_path: Union[FilePath, NeXusFile, NeXusGroup], + *, + monitor_name: MonitorName, + instrument_name: Optional[InstrumentName] = None, +) -> RawMonitor: + return RawMonitor( + _load_data( + file_path, + detector_name=monitor_name, + detector_class=snx.NXmonitor, + instrument_name=instrument_name, + ) + ) + + +def _load_data( + file_path: Union[FilePath, NeXusFile, NeXusGroup], + *, + detector_name: Union[DetectorName, MonitorName], + detector_class: Type[snx.NXobject], + instrument_name: Optional[InstrumentName] = None, +) -> sc.DataGroup: with _open_nexus_file(file_path) as f: entry = f['entry'] instrument = _unique_child_group(entry, snx.NXinstrument, instrument_name) - detector = _unique_child_group(instrument, snx.NXdetector, detector_name) - events = _unique_child_group( - detector, - snx.NXevent_data, - f'{detector_name}_events', - ) - data = events[()] - if not isinstance(data, sc.DataArray): - warnings.warn( - 'NeXus (event)data was not assembled correctly. Expected a ' - f'scipp.DataArray, but got {type(events)}.', - UserWarning, - stacklevel=2, - ) - return RawDetector(sc.DataGroup(data=data)) + detector = _unique_child_group(instrument, detector_class, detector_name) + loaded = cast(sc.DataGroup, detector[()]) + return loaded def _open_nexus_file( diff --git a/tests/nexus_test.py b/tests/nexus_test.py index b690c1cb..d414169d 100644 --- a/tests/nexus_test.py +++ b/tests/nexus_test.py @@ -31,16 +31,48 @@ def _event_data_components() -> sc.DataGroup: ) -def _write_nexus_events(store: Union[Path, BytesIO]) -> None: +def _monitor_histogram() -> sc.DataArray: + return sc.DataArray( + sc.array(dims=['time'], values=[2, 4, 8, 3], unit='counts'), + coords={ + 'time': sc.epoch(unit='ms') + + sc.array(dims=['time'], values=[2, 4, 6, 8, 10], unit='ms') + }, + ) + + +def _write_nexus_data(store: Union[Path, BytesIO]) -> None: with snx.File(store, 'w') as root: entry = root.create_class('entry', snx.NXentry) instrument = entry.create_class('reducer', snx.NXinstrument) - detector = instrument.create_class('bank12', snx.NXdetector) + detector = instrument.create_class('bank12', snx.NXdetector) events = detector.create_class('bank12_events', snx.NXevent_data) for key, val in _event_data_components().items(): events[key] = val + monitor_data = _monitor_histogram() + monitor = instrument.create_class('monitor', snx.NXmonitor) + data = monitor.create_class('data', snx.NXdata) + signal = data.create_field('signal', monitor_data.data) + signal.attrs['signal'] = 1 + signal.attrs['axes'] = monitor_data.dim + data.create_field('time', monitor_data.coords['time']) + + +# TODO more fields +""" +h5ls 60248-2022-02-28_2215.nxs/entry/instrument/larmor_detector 15:27:23 +depends_on Dataset {1} +detector_number Dataset {458752} +larmor_detector_events Group +pixel_shape Group +transformations Group +x_pixel_offset Dataset {458752} +y_pixel_offset Dataset {458752} +z_pixel_offset Dataset {458752} +""" + def _file_store(typ): if typ == Path: @@ -51,7 +83,7 @@ def _file_store(typ): @pytest.fixture(params=[Path, BytesIO, snx.Group]) def nexus_file(fs, request): store = _file_store(request.param) - _write_nexus_events(store) + _write_nexus_data(store) if isinstance(store, BytesIO): store.seek(0) @@ -86,7 +118,9 @@ def expected_bank12(): ) -# TODO histogram data +@pytest.fixture() +def expected_monitor() -> sc.DataArray: + return _monitor_histogram() @pytest.mark.parametrize('instrument_name', (None, nexus.InstrumentName('reducer'))) @@ -96,5 +130,14 @@ def test_load_detector(nexus_file, expected_bank12, instrument_name): detector_name=nexus.DetectorName('bank12'), instrument_name=instrument_name, ) - # TODO positions - sc.testing.assert_identical(detector['data'], expected_bank12) + sc.testing.assert_identical(detector['bank12_events'], expected_bank12) + + +@pytest.mark.parametrize('instrument_name', (None, nexus.InstrumentName('reducer'))) +def test_load_monitor(nexus_file, expected_monitor, instrument_name): + monitor = nexus.load_monitor( + nexus.NeXusGroup(nexus_file), + monitor_name=nexus.MonitorName('monitor'), + instrument_name=instrument_name, + ) + sc.testing.assert_identical(monitor['data'], expected_monitor) From a6237c8f0342f34d1bc4f7847ff3357a53edfe70 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Tue, 5 Mar 2024 16:30:44 +0100 Subject: [PATCH 08/25] Attempt to compute positions Failing because I can't figure out how to make scippnexus store positions in the data --- src/ess/reduce/nexus.py | 1 + tests/nexus_test.py | 47 ++++++++++++++++++++++++++--------------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/ess/reduce/nexus.py b/src/ess/reduce/nexus.py index c77480f7..9e1c9746 100644 --- a/src/ess/reduce/nexus.py +++ b/src/ess/reduce/nexus.py @@ -82,6 +82,7 @@ def _load_data( instrument = _unique_child_group(entry, snx.NXinstrument, instrument_name) detector = _unique_child_group(instrument, detector_class, detector_name) loaded = cast(sc.DataGroup, detector[()]) + loaded = snx.compute_positions(loaded) return loaded diff --git a/tests/nexus_test.py b/tests/nexus_test.py index d414169d..342e215e 100644 --- a/tests/nexus_test.py +++ b/tests/nexus_test.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Union +import numpy as np import pytest import scipp as sc import scipp.testing @@ -27,6 +28,10 @@ def _event_data_components() -> sc.DataGroup: 'event_index': sc.array( dims=['event_time_zero'], unit=None, values=[0, 3, 3, 6] ), + 'offset': sc.vector([0.4, 0.0, 11.5], unit='m'), + 'pixel_offset': sc.vectors( + dims=['event'], values=np.arange(3 * 6).reshape((6, 3)), unit='m' + ), } ) @@ -36,11 +41,23 @@ def _monitor_histogram() -> sc.DataArray: sc.array(dims=['time'], values=[2, 4, 8, 3], unit='counts'), coords={ 'time': sc.epoch(unit='ms') - + sc.array(dims=['time'], values=[2, 4, 6, 8, 10], unit='ms') + + sc.array(dims=['time'], values=[2, 4, 6, 8, 10], unit='ms'), + 'position': sc.vector([0.1, -0.4, 10.3], unit='m'), }, ) +def _write_transformation(group: snx.Group, offset: sc.Variable) -> None: + group.create_field('depends_on', sc.scalar('transformations/t1')) + transformations = group.create_class('transformations', snx.NXtransformations) + t1 = transformations.create_field('t1', sc.scalar(0.0, unit=offset.unit)) + t1.attrs['depends_on'] = '.' + t1.attrs['transformation_type'] = 'translation' + t1.attrs['offset'] = offset.values + t1.attrs['offset_units'] = str(offset.unit) + t1.attrs['vector'] = sc.vector([0, 0, 1]).value + + def _write_nexus_data(store: Union[Path, BytesIO]) -> None: with snx.File(store, 'w') as root: entry = root.create_class('entry', snx.NXentry) @@ -48,8 +65,15 @@ def _write_nexus_data(store: Union[Path, BytesIO]) -> None: detector = instrument.create_class('bank12', snx.NXdetector) events = detector.create_class('bank12_events', snx.NXevent_data) - for key, val in _event_data_components().items(): - events[key] = val + detector_components = _event_data_components() + events['event_id'] = detector_components['event_id'] + events['event_time_offset'] = detector_components['event_time_offset'] + events['event_time_zero'] = detector_components['event_time_zero'] + events['event_index'] = detector_components['event_index'] + events['x_pixel_offset'] = detector_components['pixel_offset'].fields.x + events['y_pixel_offset'] = detector_components['pixel_offset'].fields.y + events['z_pixel_offset'] = detector_components['pixel_offset'].fields.z + _write_transformation(detector, detector_components['offset']) monitor_data = _monitor_histogram() monitor = instrument.create_class('monitor', snx.NXmonitor) @@ -58,20 +82,7 @@ def _write_nexus_data(store: Union[Path, BytesIO]) -> None: signal.attrs['signal'] = 1 signal.attrs['axes'] = monitor_data.dim data.create_field('time', monitor_data.coords['time']) - - -# TODO more fields -""" -h5ls 60248-2022-02-28_2215.nxs/entry/instrument/larmor_detector 15:27:23 -depends_on Dataset {1} -detector_number Dataset {458752} -larmor_detector_events Group -pixel_shape Group -transformations Group -x_pixel_offset Dataset {458752} -y_pixel_offset Dataset {458752} -z_pixel_offset Dataset {458752} -""" + _write_transformation(monitor, monitor_data.coords['position']) def _file_store(typ): @@ -94,6 +105,7 @@ def nexus_file(fs, request): yield f +# TODO larmor data is grouped by pixel, not binned in time @pytest.fixture() def expected_bank12(): components = _event_data_components() @@ -102,6 +114,7 @@ def expected_bank12(): coords={ 'event_id': components['event_id'], 'event_time_offset': components['event_time_offset'], + 'position': components['offset'] + components['pixel_offset'], }, ) events = sc.bins( From 4bc417ab044993aee53ec8dd10509568e2aca026 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 6 Mar 2024 10:29:47 +0100 Subject: [PATCH 09/25] Detector test data with pixel binning --- tests/nexus_test.py | 58 ++++++++++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/tests/nexus_test.py b/tests/nexus_test.py index 342e215e..acb7784e 100644 --- a/tests/nexus_test.py +++ b/tests/nexus_test.py @@ -22,15 +22,17 @@ def _event_data_components() -> sc.DataGroup: 'event_time_offset': sc.array( dims=['event'], unit='s', values=[456, 7, 3, 345, 632, 23] ), - 'event_time_zero': sc.array( - dims=['event_time_zero'], unit='s', values=[1, 2, 3, 4] - ), + 'event_time_zero': sc.epoch(unit='s') + + sc.array(dims=['event_index'], unit='s', values=[1, 2, 3, 4]), 'event_index': sc.array( - dims=['event_time_zero'], unit=None, values=[0, 3, 3, 6] + dims=['event_index'], unit=None, values=[0, 3, 3, 6] ), + 'detector_number': sc.arange('detector_number', 5, unit=None), 'offset': sc.vector([0.4, 0.0, 11.5], unit='m'), 'pixel_offset': sc.vectors( - dims=['event'], values=np.arange(3 * 6).reshape((6, 3)), unit='m' + dims=['detector_number'], + values=np.arange(3 * 5).reshape((5, 3)), + unit='m', ), } ) @@ -70,9 +72,10 @@ def _write_nexus_data(store: Union[Path, BytesIO]) -> None: events['event_time_offset'] = detector_components['event_time_offset'] events['event_time_zero'] = detector_components['event_time_zero'] events['event_index'] = detector_components['event_index'] - events['x_pixel_offset'] = detector_components['pixel_offset'].fields.x - events['y_pixel_offset'] = detector_components['pixel_offset'].fields.y - events['z_pixel_offset'] = detector_components['pixel_offset'].fields.z + detector['x_pixel_offset'] = detector_components['pixel_offset'].fields.x + detector['y_pixel_offset'] = detector_components['pixel_offset'].fields.y + detector['z_pixel_offset'] = detector_components['pixel_offset'].fields.z + detector['detector_number'] = detector_components['detector_number'] _write_transformation(detector, detector_components['offset']) monitor_data = _monitor_histogram() @@ -105,31 +108,42 @@ def nexus_file(fs, request): yield f -# TODO larmor data is grouped by pixel, not binned in time @pytest.fixture() def expected_bank12(): components = _event_data_components() buffer = sc.DataArray( - sc.ones(sizes={'event': 6}, unit='counts'), + sc.ones(sizes={'event': 6}, unit='counts', dtype='float32'), coords={ - 'event_id': components['event_id'], + 'detector_number': components['event_id'], 'event_time_offset': components['event_time_offset'], - 'position': components['offset'] + components['pixel_offset'], }, ) - events = sc.bins( - data=buffer, - begin=components['event_index'], - end=sc.concat( - [components['event_index'][1:], components['event_index'][-1]], - dim='event_time_zero', - ), - dim='event', + + # Bin by event_index tp broadcast event_time_zero to events + binned_in_time = sc.DataArray( + sc.bins( + data=buffer, + begin=components['event_index'], + end=sc.concat( + [components['event_index'][1:], components['event_index'][-1]], + dim='event_index', + ), + dim='event', + ) ) - return sc.DataArray( - events, coords={'event_time_zero': components['event_time_zero']} + binned_in_time.bins.coords['event_time_zero'] = sc.bins_like( + binned_in_time, components['event_time_zero'] ) + # Bin by detector number like ScippNexus would + binned = binned_in_time.bins.concat().group(components['detector_number']) + binned.coords['x_pixel_offset'] = components['pixel_offset'].fields.x + binned.coords['y_pixel_offset'] = components['pixel_offset'].fields.y + binned.coords['z_pixel_offset'] = components['pixel_offset'].fields.z + # Computed position + binned.coords['position'] = components['offset'] + components['pixel_offset'] + return binned + @pytest.fixture() def expected_monitor() -> sc.DataArray: From c859a3786ab0eae6f2c5c1df536d78a088f30c99 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 6 Mar 2024 10:40:02 +0100 Subject: [PATCH 10/25] Do not store position for monitor --- tests/nexus_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/nexus_test.py b/tests/nexus_test.py index acb7784e..51d3fcd6 100644 --- a/tests/nexus_test.py +++ b/tests/nexus_test.py @@ -44,7 +44,6 @@ def _monitor_histogram() -> sc.DataArray: coords={ 'time': sc.epoch(unit='ms') + sc.array(dims=['time'], values=[2, 4, 6, 8, 10], unit='ms'), - 'position': sc.vector([0.1, -0.4, 10.3], unit='m'), }, ) @@ -85,7 +84,6 @@ def _write_nexus_data(store: Union[Path, BytesIO]) -> None: signal.attrs['signal'] = 1 signal.attrs['axes'] = monitor_data.dim data.create_field('time', monitor_data.coords['time']) - _write_transformation(monitor, monitor_data.coords['position']) def _file_store(typ): From 2f614d5fea61d448d0a0f4718894d0569b3246db Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 6 Mar 2024 13:07:38 +0100 Subject: [PATCH 11/25] Add load_source --- src/ess/reduce/nexus.py | 53 +++++++++++++++++++++++++++++++---------- tests/nexus_test.py | 39 ++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 13 deletions(-) diff --git a/src/ess/reduce/nexus.py b/src/ess/reduce/nexus.py index 9e1c9746..50a17d9a 100644 --- a/src/ess/reduce/nexus.py +++ b/src/ess/reduce/nexus.py @@ -22,17 +22,23 @@ NeXusGroup = NewType('NeXusGroup', snx.Group) """A ScippNexus group in an open file.""" -InstrumentName = NewType('InstrumentName', str) -"""Name of an instrument in a NeXus file.""" DetectorName = NewType('DetectorName', str) """Name of a detector (bank) in a NeXus file.""" +InstrumentName = NewType('InstrumentName', str) +"""Name of an instrument in a NeXus file.""" MonitorName = NewType('MonitorName', str) """Name of a monitor in a NeXus file.""" +SourceName = NewType('SourceName', str) +"""Name of a source in a NeXus file.""" RawDetector = NewType('RawDetector', sc.DataGroup) """Raw data from a NeXus detector.""" RawMonitor = NewType('RawMonitor', sc.DataGroup) """Raw data from a NeXus monitor.""" +RawSample = NewType('RawSample', sc.DataGroup) +"""Raw data from a NeXus sample.""" +RawSource = NewType('RawSource', sc.DataGroup) +"""Raw data from a NeXus source.""" def load_detector( @@ -45,10 +51,10 @@ def load_detector( TODO handling of names, including event name """ return RawDetector( - _load_data( + _load_group_with_positions( file_path, - detector_name=detector_name, - detector_class=snx.NXdetector, + group_name=detector_name, + nx_class=snx.NXdetector, instrument_name=instrument_name, ) ) @@ -61,27 +67,44 @@ def load_monitor( instrument_name: Optional[InstrumentName] = None, ) -> RawMonitor: return RawMonitor( - _load_data( + _load_group_with_positions( + file_path, + group_name=monitor_name, + nx_class=snx.NXmonitor, + instrument_name=instrument_name, + ) + ) + + +def load_source( + file_path: Union[FilePath, NeXusFile, NeXusGroup], + *, + source_name: Optional[SourceName] = None, + instrument_name: Optional[InstrumentName] = None, +) -> RawSource: + return RawSource( + _load_group_with_positions( file_path, - detector_name=monitor_name, - detector_class=snx.NXmonitor, + group_name=source_name, + nx_class=snx.NXsource, instrument_name=instrument_name, ) ) -def _load_data( +def _load_group_with_positions( file_path: Union[FilePath, NeXusFile, NeXusGroup], *, - detector_name: Union[DetectorName, MonitorName], - detector_class: Type[snx.NXobject], + group_name: Optional[str], + nx_class: Type[snx.NXobject], instrument_name: Optional[InstrumentName] = None, ) -> sc.DataGroup: with _open_nexus_file(file_path) as f: entry = f['entry'] instrument = _unique_child_group(entry, snx.NXinstrument, instrument_name) - detector = _unique_child_group(instrument, detector_class, detector_name) - loaded = cast(sc.DataGroup, detector[()]) + loaded = cast( + sc.DataGroup, _unique_child_group(instrument, nx_class, group_name)[()] + ) loaded = snx.compute_positions(loaded) return loaded @@ -114,3 +137,7 @@ def _unique_child_group( if len(children) != 1: raise ValueError(f'Expected exactly one {nx_class} group, got {len(children)}') return next(iter(children.values())) # type: ignore[return-value] + + +# TODO source +# TODO sample diff --git a/tests/nexus_test.py b/tests/nexus_test.py index 51d3fcd6..9e2ce809 100644 --- a/tests/nexus_test.py +++ b/tests/nexus_test.py @@ -48,6 +48,17 @@ def _monitor_histogram() -> sc.DataArray: ) +def _source_data() -> sc.DataGroup: + return sc.DataGroup( + { + 'name': 'moderator', + 'probe': 'neutron', + 'type': 'Spallation Neutron Source', + 'position': sc.vector([0, 0, 0], unit='m'), + } + ) + + def _write_transformation(group: snx.Group, offset: sc.Variable) -> None: group.create_field('depends_on', sc.scalar('transformations/t1')) transformations = group.create_class('transformations', snx.NXtransformations) @@ -85,6 +96,13 @@ def _write_nexus_data(store: Union[Path, BytesIO]) -> None: signal.attrs['axes'] = monitor_data.dim data.create_field('time', monitor_data.coords['time']) + source_data = _source_data() + source = instrument.create_class('source', snx.NXsource) + source.create_field('name', source_data['name']) + source.create_field('probe', source_data['probe']) + source.create_field('type', source_data['type']) + _write_transformation(source, source_data['position']) + def _file_store(typ): if typ == Path: @@ -94,6 +112,8 @@ def _file_store(typ): @pytest.fixture(params=[Path, BytesIO, snx.Group]) def nexus_file(fs, request): + _ = fs # Request fs to prevent writing to real disk when param=Path + store = _file_store(request.param) _write_nexus_data(store) if isinstance(store, BytesIO): @@ -148,6 +168,11 @@ def expected_monitor() -> sc.DataArray: return _monitor_histogram() +@pytest.fixture() +def expected_source() -> sc.DataGroup: + return _source_data() + + @pytest.mark.parametrize('instrument_name', (None, nexus.InstrumentName('reducer'))) def test_load_detector(nexus_file, expected_bank12, instrument_name): detector = nexus.load_detector( @@ -166,3 +191,17 @@ def test_load_monitor(nexus_file, expected_monitor, instrument_name): instrument_name=instrument_name, ) sc.testing.assert_identical(monitor['data'], expected_monitor) + + +@pytest.mark.parametrize('instrument_name', (None, nexus.InstrumentName('reducer'))) +@pytest.mark.parametrize('source_name', (None, nexus.SourceName('source'))) +def test_load_source(nexus_file, expected_source, instrument_name, source_name): + source = nexus.load_source( + nexus.NeXusGroup(nexus_file), + instrument_name=instrument_name, + source_name=source_name, + ) + # NeXus details that we don't need to test as long as the positions are ok: + del source['depends_on'] + del source['transformations'] + sc.testing.assert_identical(source, nexus.RawSource(expected_source)) From 43bc9915f3f6c2b341b7887034f6a43ae180fac7 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 6 Mar 2024 13:18:12 +0100 Subject: [PATCH 12/25] Add load_sample --- src/ess/reduce/nexus.py | 13 +++++++++---- tests/nexus_test.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/ess/reduce/nexus.py b/src/ess/reduce/nexus.py index 50a17d9a..6a6bbb90 100644 --- a/src/ess/reduce/nexus.py +++ b/src/ess/reduce/nexus.py @@ -92,6 +92,15 @@ def load_source( ) +def load_sample( + file_path: Union[FilePath, NeXusFile, NeXusGroup], +) -> RawSample: + with _open_nexus_file(file_path) as f: + entry = f['entry'] + loaded = cast(sc.DataGroup, _unique_child_group(entry, snx.NXsample, None)[()]) + return RawSample(loaded) + + def _load_group_with_positions( file_path: Union[FilePath, NeXusFile, NeXusGroup], *, @@ -137,7 +146,3 @@ def _unique_child_group( if len(children) != 1: raise ValueError(f'Expected exactly one {nx_class} group, got {len(children)}') return next(iter(children.values())) # type: ignore[return-value] - - -# TODO source -# TODO sample diff --git a/tests/nexus_test.py b/tests/nexus_test.py index 9e2ce809..decfe467 100644 --- a/tests/nexus_test.py +++ b/tests/nexus_test.py @@ -59,6 +59,16 @@ def _source_data() -> sc.DataGroup: ) +def _sample_data() -> sc.DataGroup: + return sc.DataGroup( + { + 'name': 'water', + 'chemical_formula': 'H2O', + 'type': 'sample+can', + } + ) + + def _write_transformation(group: snx.Group, offset: sc.Variable) -> None: group.create_field('depends_on', sc.scalar('transformations/t1')) transformations = group.create_class('transformations', snx.NXtransformations) @@ -103,6 +113,12 @@ def _write_nexus_data(store: Union[Path, BytesIO]) -> None: source.create_field('type', source_data['type']) _write_transformation(source, source_data['position']) + sample_data = _sample_data() + sample = entry.create_class('sample', snx.NXsample) + sample.create_field('name', sample_data['name']) + sample.create_field('chemical_formula', sample_data['chemical_formula']) + sample.create_field('type', sample_data['type']) + def _file_store(typ): if typ == Path: @@ -173,6 +189,11 @@ def expected_source() -> sc.DataGroup: return _source_data() +@pytest.fixture() +def expected_sample() -> sc.DataGroup: + return _sample_data() + + @pytest.mark.parametrize('instrument_name', (None, nexus.InstrumentName('reducer'))) def test_load_detector(nexus_file, expected_bank12, instrument_name): detector = nexus.load_detector( @@ -205,3 +226,10 @@ def test_load_source(nexus_file, expected_source, instrument_name, source_name): del source['depends_on'] del source['transformations'] sc.testing.assert_identical(source, nexus.RawSource(expected_source)) + + +def test_load_sample(nexus_file, expected_sample): + sample = nexus.load_sample( + nexus.NeXusGroup(nexus_file), + ) + sc.testing.assert_identical(sample, nexus.RawSample(expected_sample)) From 5fbd9168847169ee25e3d07527e5fc58668eddc1 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 6 Mar 2024 13:32:12 +0100 Subject: [PATCH 13/25] Add data extraction functions --- src/ess/reduce/nexus.py | 34 ++++++++++++++++++++++++++++++++-- tests/nexus_test.py | 24 ++++++++++++++++++------ 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/src/ess/reduce/nexus.py b/src/ess/reduce/nexus.py index 6a6bbb90..a4f28bea 100644 --- a/src/ess/reduce/nexus.py +++ b/src/ess/reduce/nexus.py @@ -32,9 +32,13 @@ """Name of a source in a NeXus file.""" RawDetector = NewType('RawDetector', sc.DataGroup) -"""Raw data from a NeXus detector.""" +"""Full raw data from a NeXus detector.""" +RawDetectorData = NewType('RawDetectorData', sc.DataArray) +"""Data extracted from a RawDetector.""" RawMonitor = NewType('RawMonitor', sc.DataGroup) -"""Raw data from a NeXus monitor.""" +"""Full raw data from a NeXus monitor.""" +RawMonitorData = NewType('RawMonitorData', sc.DataArray) +"""Data extracted from a RawMonitor.""" RawSample = NewType('RawSample', sc.DataGroup) """Raw data from a NeXus sample.""" RawSource = NewType('RawSource', sc.DataGroup) @@ -146,3 +150,29 @@ def _unique_child_group( if len(children) != 1: raise ValueError(f'Expected exactly one {nx_class} group, got {len(children)}') return next(iter(children.values())) # type: ignore[return-value] + + +def extract_detector_data( + detector: RawDetector, detector_name: DetectorName +) -> RawDetectorData: + return RawDetectorData(_extract_events_or_histogram(detector, detector_name)) + + +def extract_monitor_data( + monitor: RawMonitor, monitor_name: MonitorName +) -> RawMonitorData: + return RawMonitorData(_extract_events_or_histogram(monitor, monitor_name)) + + +def _extract_events_or_histogram(dg: sc.DataGroup, name: str) -> sc.DataArray: + data_names = {f'{name}_events', 'data'} + for data_name in data_names: + try: + return dg[data_name] + except KeyError: + pass + raise ValueError( + f"Raw data '{name}' loaded from NeXus does not contain events or a histogram. " + f"Expected to find one of {data_names}, " + f"but the data only contains {set(dg.keys())}" + ) diff --git a/tests/nexus_test.py b/tests/nexus_test.py index decfe467..6c5ac16a 100644 --- a/tests/nexus_test.py +++ b/tests/nexus_test.py @@ -197,7 +197,7 @@ def expected_sample() -> sc.DataGroup: @pytest.mark.parametrize('instrument_name', (None, nexus.InstrumentName('reducer'))) def test_load_detector(nexus_file, expected_bank12, instrument_name): detector = nexus.load_detector( - nexus.NeXusGroup(nexus_file), + nexus_file, detector_name=nexus.DetectorName('bank12'), instrument_name=instrument_name, ) @@ -207,7 +207,7 @@ def test_load_detector(nexus_file, expected_bank12, instrument_name): @pytest.mark.parametrize('instrument_name', (None, nexus.InstrumentName('reducer'))) def test_load_monitor(nexus_file, expected_monitor, instrument_name): monitor = nexus.load_monitor( - nexus.NeXusGroup(nexus_file), + nexus_file, monitor_name=nexus.MonitorName('monitor'), instrument_name=instrument_name, ) @@ -218,7 +218,7 @@ def test_load_monitor(nexus_file, expected_monitor, instrument_name): @pytest.mark.parametrize('source_name', (None, nexus.SourceName('source'))) def test_load_source(nexus_file, expected_source, instrument_name, source_name): source = nexus.load_source( - nexus.NeXusGroup(nexus_file), + nexus_file, instrument_name=instrument_name, source_name=source_name, ) @@ -229,7 +229,19 @@ def test_load_source(nexus_file, expected_source, instrument_name, source_name): def test_load_sample(nexus_file, expected_sample): - sample = nexus.load_sample( - nexus.NeXusGroup(nexus_file), - ) + sample = nexus.load_sample(nexus_file) sc.testing.assert_identical(sample, nexus.RawSample(expected_sample)) + + +def test_extract_detector_data(nexus_file, expected_bank12): + detector_name = nexus.DetectorName('bank12') + detector = nexus.load_detector(nexus_file, detector_name=detector_name) + data = nexus.extract_detector_data(detector, detector_name=detector_name) + sc.testing.assert_identical(data, nexus.RawDetectorData(expected_bank12)) + + +def test_extract_monitor_data(nexus_file, expected_monitor): + monitor_name = nexus.MonitorName('monitor') + monitor = nexus.load_monitor(nexus_file, monitor_name=monitor_name) + data = nexus.extract_monitor_data(monitor, monitor_name=monitor_name) + sc.testing.assert_identical(data, nexus.RawMonitorData(expected_monitor)) From 0f76d4c491e41a73556cfe5188900c223dbf9bf4 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 6 Mar 2024 13:49:44 +0100 Subject: [PATCH 14/25] Document nexus utilities --- docs/api-reference/index.md | 2 + src/ess/reduce/nexus.py | 155 +++++++++++++++++++++++++++++++++++- 2 files changed, 155 insertions(+), 2 deletions(-) diff --git a/docs/api-reference/index.md b/docs/api-reference/index.md index 70d4d966..04e8cf95 100644 --- a/docs/api-reference/index.md +++ b/docs/api-reference/index.md @@ -26,4 +26,6 @@ :toctree: ../generated/modules :template: module-template.rst :recursive: + + nexus ``` diff --git a/src/ess/reduce/nexus.py b/src/ess/reduce/nexus.py index a4f28bea..4b022168 100644 --- a/src/ess/reduce/nexus.py +++ b/src/ess/reduce/nexus.py @@ -1,6 +1,15 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +"""NeXus utilities. + +This module defines functions and domain types that can be used +to build Sciline pipelines for simple workflows. +If multiple different kind sof files (e.g., sample and background runs) +are needed, custom types and providers need to be defined to wrap +the basic ones here. +""" + from contextlib import nullcontext from pathlib import Path from typing import BinaryIO, ContextManager, NewType, Optional, Type, Union, cast @@ -51,8 +60,32 @@ def load_detector( detector_name: DetectorName, instrument_name: Optional[InstrumentName] = None, ) -> RawDetector: - """ - TODO handling of names, including event name + """Load a single detector (bank) from a NeXus file. + + The detector positions are computed automatically for NeXus transformations. + + Parameters + ---------- + file_path: + Indicates where to load data from. + One of: + + - Path to a NeXus file on disk. + - File handle or buffer for reading binary data. + - A ScippNexus group of the root of a NeXus file. + detector_name: + Name of the detector (bank) to load. + Must be a group in the instrument group (see below). + instrument_name: + Name of the instrument that contains the detector. + If ``None``, the instrument will be located based + on its NeXus class. + + Returns + ------- + : + A data group containing the detector events or histogram + and any auxiliary data stored in the same NeXus group. """ return RawDetector( _load_group_with_positions( @@ -70,6 +103,33 @@ def load_monitor( monitor_name: MonitorName, instrument_name: Optional[InstrumentName] = None, ) -> RawMonitor: + """Load a single monitor from a NeXus file. + + The monitor position is computed automatically for NeXus transformations. + + Parameters + ---------- + file_path: + Indicates where to load data from. + One of: + + - Path to a NeXus file on disk. + - File handle or buffer for reading binary data. + - A ScippNexus group of the root of a NeXus file. + monitor_name: + Name of the monitor to load. + Must be a group in the instrument group (see below). + instrument_name: + Name of the instrument that contains the detector. + If ``None``, the instrument will be located based + on its NeXus class. + + Returns + ------- + : + A data group containing the monitor events or histogram + and any auxiliary data stored in the same NeXus group. + """ return RawMonitor( _load_group_with_positions( file_path, @@ -86,6 +146,35 @@ def load_source( source_name: Optional[SourceName] = None, instrument_name: Optional[InstrumentName] = None, ) -> RawSource: + """Load a source from a NeXus file. + + The source position is computed automatically for NeXus transformations. + + Parameters + ---------- + file_path: + Indicates where to load data from. + One of: + + - Path to a NeXus file on disk. + - File handle or buffer for reading binary data. + - A ScippNexus group of the root of a NeXus file. + source_name: + Name of the source to load. + Must be a group in the instrument group (see below). + If ``None``, the source will be located based + on its NeXus class. + instrument_name: + Name of the instrument that contains the detector. + If ``None``, the instrument will be located based + on its NeXus class. + + Returns + ------- + : + A data group containing all data stored in + the source NeXus group. + """ return RawSource( _load_group_with_positions( file_path, @@ -99,6 +188,28 @@ def load_source( def load_sample( file_path: Union[FilePath, NeXusFile, NeXusGroup], ) -> RawSample: + """Load a sample from a NeXus file. + + The sample is located based on its NeXus class. + There can be only one sample in a NeXus file or + in the group given as ``file_path``. + + Parameters + ---------- + file_path: + Indicates where to load data from. + One of: + + - Path to a NeXus file on disk. + - File handle or buffer for reading binary data. + - A ScippNexus group of the root of a NeXus file. + + Returns + ------- + : + A data group containing all data stored in + the sample NeXus group. + """ with _open_nexus_file(file_path) as f: entry = f['entry'] loaded = cast(sc.DataGroup, _unique_child_group(entry, snx.NXsample, None)[()]) @@ -155,12 +266,52 @@ def _unique_child_group( def extract_detector_data( detector: RawDetector, detector_name: DetectorName ) -> RawDetectorData: + """Get and return the events or histogram from a detector loaded from NeXus. + + Parameters + ---------- + detector: + A detector loaded from NeXus. + detector_name: + Name of the detector. + + Returns + ------- + : + A data array containing the events or histogram. + + See also + -------- + load_detector: + Load a detector from a NeXus file in a format compatible with + ``extract_detector_data``. + """ return RawDetectorData(_extract_events_or_histogram(detector, detector_name)) def extract_monitor_data( monitor: RawMonitor, monitor_name: MonitorName ) -> RawMonitorData: + """Get and return the events or histogram from a monitor loaded from NeXus. + + Parameters + ---------- + monitor: + A monitor loaded from NeXus. + monitor_name: + Name of the monitor. + + Returns + ------- + : + A data array containing the events or histogram. + + See also + -------- + load_monitor: + Load a monitor from a NeXus file in a format compatible with + ``extract_monitor_data``. + """ return RawMonitorData(_extract_events_or_histogram(monitor, monitor_name)) From 3021dd35544e25539d3eda021cbdba9f7b1e6cc9 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 6 Mar 2024 14:13:36 +0100 Subject: [PATCH 15/25] Parametrize entry name instead instrument name --- src/ess/reduce/nexus.py | 65 ++++++++++++++++++++++------------------- tests/nexus_test.py | 23 ++++++++------- 2 files changed, 47 insertions(+), 41 deletions(-) diff --git a/src/ess/reduce/nexus.py b/src/ess/reduce/nexus.py index 4b022168..b410f398 100644 --- a/src/ess/reduce/nexus.py +++ b/src/ess/reduce/nexus.py @@ -33,8 +33,8 @@ DetectorName = NewType('DetectorName', str) """Name of a detector (bank) in a NeXus file.""" -InstrumentName = NewType('InstrumentName', str) -"""Name of an instrument in a NeXus file.""" +EntryName = NewType('EntryName', str) +"""Name of an entry in a NeXus file.""" MonitorName = NewType('MonitorName', str) """Name of a monitor in a NeXus file.""" SourceName = NewType('SourceName', str) @@ -58,7 +58,7 @@ def load_detector( file_path: Union[FilePath, NeXusFile, NeXusGroup], *, detector_name: DetectorName, - instrument_name: Optional[InstrumentName] = None, + entry_name: Optional[EntryName] = None, ) -> RawDetector: """Load a single detector (bank) from a NeXus file. @@ -75,11 +75,11 @@ def load_detector( - A ScippNexus group of the root of a NeXus file. detector_name: Name of the detector (bank) to load. - Must be a group in the instrument group (see below). - instrument_name: - Name of the instrument that contains the detector. - If ``None``, the instrument will be located based - on its NeXus class. + Must be a group in an instrument group in the entry (see below). + entry_name: + Name of the entry that contains the detector. + If ``None``, the entry will be located based + on its NeXus class, but there cannot be more than 1. Returns ------- @@ -92,7 +92,7 @@ def load_detector( file_path, group_name=detector_name, nx_class=snx.NXdetector, - instrument_name=instrument_name, + entry_name=entry_name, ) ) @@ -101,7 +101,7 @@ def load_monitor( file_path: Union[FilePath, NeXusFile, NeXusGroup], *, monitor_name: MonitorName, - instrument_name: Optional[InstrumentName] = None, + entry_name: Optional[EntryName] = None, ) -> RawMonitor: """Load a single monitor from a NeXus file. @@ -118,11 +118,11 @@ def load_monitor( - A ScippNexus group of the root of a NeXus file. monitor_name: Name of the monitor to load. - Must be a group in the instrument group (see below). - instrument_name: - Name of the instrument that contains the detector. - If ``None``, the instrument will be located based - on its NeXus class. + Must be a group in an instrument group in the entry (see below). + entry_name: + Name of the entry that contains the monitor. + If ``None``, the entry will be located based + on its NeXus class, but there cannot be more than 1. Returns ------- @@ -135,7 +135,7 @@ def load_monitor( file_path, group_name=monitor_name, nx_class=snx.NXmonitor, - instrument_name=instrument_name, + entry_name=entry_name, ) ) @@ -144,7 +144,7 @@ def load_source( file_path: Union[FilePath, NeXusFile, NeXusGroup], *, source_name: Optional[SourceName] = None, - instrument_name: Optional[InstrumentName] = None, + entry_name: Optional[EntryName] = None, ) -> RawSource: """Load a source from a NeXus file. @@ -161,38 +161,39 @@ def load_source( - A ScippNexus group of the root of a NeXus file. source_name: Name of the source to load. - Must be a group in the instrument group (see below). + Must be a group in an instrument group in the entry (see below). If ``None``, the source will be located based on its NeXus class. - instrument_name: - Name of the instrument that contains the detector. - If ``None``, the instrument will be located based - on its NeXus class. + entry_name: + Name of the instrument that contains the source. + If ``None``, the entry will be located based + on its NeXus class, but there cannot be more than 1. Returns ------- : A data group containing all data stored in - the source NeXus group. + the source NeXus group. """ return RawSource( _load_group_with_positions( file_path, group_name=source_name, nx_class=snx.NXsource, - instrument_name=instrument_name, + entry_name=entry_name, ) ) def load_sample( file_path: Union[FilePath, NeXusFile, NeXusGroup], + entry_name: Optional[EntryName] = None, ) -> RawSample: """Load a sample from a NeXus file. The sample is located based on its NeXus class. There can be only one sample in a NeXus file or - in the group given as ``file_path``. + in the entry indicated by ``entry_name``. Parameters ---------- @@ -203,15 +204,19 @@ def load_sample( - Path to a NeXus file on disk. - File handle or buffer for reading binary data. - A ScippNexus group of the root of a NeXus file. + entry_name: + Name of the instrument that contains the source. + If ``None``, the entry will be located based + on its NeXus class, but there cannot be more than 1. Returns ------- : A data group containing all data stored in - the sample NeXus group. + the sample NeXus group. """ with _open_nexus_file(file_path) as f: - entry = f['entry'] + entry = _unique_child_group(f, snx.NXentry, entry_name) loaded = cast(sc.DataGroup, _unique_child_group(entry, snx.NXsample, None)[()]) return RawSample(loaded) @@ -221,11 +226,11 @@ def _load_group_with_positions( *, group_name: Optional[str], nx_class: Type[snx.NXobject], - instrument_name: Optional[InstrumentName] = None, + entry_name: Optional[EntryName] = None, ) -> sc.DataGroup: with _open_nexus_file(file_path) as f: - entry = f['entry'] - instrument = _unique_child_group(entry, snx.NXinstrument, instrument_name) + entry = _unique_child_group(f, snx.NXentry, entry_name) + instrument = _unique_child_group(entry, snx.NXinstrument, None) loaded = cast( sc.DataGroup, _unique_child_group(instrument, nx_class, group_name)[()] ) diff --git a/tests/nexus_test.py b/tests/nexus_test.py index 6c5ac16a..8a8d59b3 100644 --- a/tests/nexus_test.py +++ b/tests/nexus_test.py @@ -194,32 +194,32 @@ def expected_sample() -> sc.DataGroup: return _sample_data() -@pytest.mark.parametrize('instrument_name', (None, nexus.InstrumentName('reducer'))) -def test_load_detector(nexus_file, expected_bank12, instrument_name): +@pytest.mark.parametrize('entry_name', (None, nexus.EntryName('entry'))) +def test_load_detector(nexus_file, expected_bank12, entry_name): detector = nexus.load_detector( nexus_file, detector_name=nexus.DetectorName('bank12'), - instrument_name=instrument_name, + entry_name=entry_name, ) sc.testing.assert_identical(detector['bank12_events'], expected_bank12) -@pytest.mark.parametrize('instrument_name', (None, nexus.InstrumentName('reducer'))) -def test_load_monitor(nexus_file, expected_monitor, instrument_name): +@pytest.mark.parametrize('entry_name', (None, nexus.EntryName('entry'))) +def test_load_monitor(nexus_file, expected_monitor, entry_name): monitor = nexus.load_monitor( nexus_file, monitor_name=nexus.MonitorName('monitor'), - instrument_name=instrument_name, + entry_name=entry_name, ) sc.testing.assert_identical(monitor['data'], expected_monitor) -@pytest.mark.parametrize('instrument_name', (None, nexus.InstrumentName('reducer'))) +@pytest.mark.parametrize('entry_name', (None, nexus.EntryName('entry'))) @pytest.mark.parametrize('source_name', (None, nexus.SourceName('source'))) -def test_load_source(nexus_file, expected_source, instrument_name, source_name): +def test_load_source(nexus_file, expected_source, entry_name, source_name): source = nexus.load_source( nexus_file, - instrument_name=instrument_name, + entry_name=entry_name, source_name=source_name, ) # NeXus details that we don't need to test as long as the positions are ok: @@ -228,8 +228,9 @@ def test_load_source(nexus_file, expected_source, instrument_name, source_name): sc.testing.assert_identical(source, nexus.RawSource(expected_source)) -def test_load_sample(nexus_file, expected_sample): - sample = nexus.load_sample(nexus_file) +@pytest.mark.parametrize('entry_name', (None, nexus.EntryName('entry'))) +def test_load_sample(nexus_file, expected_sample, entry_name): + sample = nexus.load_sample(nexus_file, entry_name=entry_name) sc.testing.assert_identical(sample, nexus.RawSample(expected_sample)) From 1358ff1310f6680b8b95182e389a4b1f54b8c37c Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 6 Mar 2024 16:00:09 +0100 Subject: [PATCH 16/25] Use temp dir instead of pyfakefs --- conda/meta.yaml | 1 - requirements/basetest.in | 1 - requirements/basetest.txt | 4 +--- tests/nexus_test.py | 39 +++++++++++++++++++++------------------ 4 files changed, 22 insertions(+), 23 deletions(-) diff --git a/conda/meta.yaml b/conda/meta.yaml index 745b286b..96571c25 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -19,7 +19,6 @@ test: imports: - ess.reduce requires: - - pyfakefs - pytest source_files: - pyproject.toml diff --git a/requirements/basetest.in b/requirements/basetest.in index a97ea118..e4a48b29 100644 --- a/requirements/basetest.in +++ b/requirements/basetest.in @@ -1,5 +1,4 @@ # Dependencies that are only used by tests. # Do not make an environment from this file, use test.txt instead! -pyfakefs pytest diff --git a/requirements/basetest.txt b/requirements/basetest.txt index 9b28aa74..a1fc27b6 100644 --- a/requirements/basetest.txt +++ b/requirements/basetest.txt @@ -1,4 +1,4 @@ -# SHA1:81a76547a01dccca58b7135a0696f0d92bc197c3 +# SHA1:0eaa389e1fdb3a1917c0f987514bd561be5718ee # # This file is autogenerated by pip-compile-multi # To update, run: @@ -13,8 +13,6 @@ packaging==23.2 # via pytest pluggy==1.4.0 # via pytest -pyfakefs==5.3.5 - # via -r basetest.in pytest==8.0.2 # via -r basetest.in tomli==2.0.1 diff --git a/tests/nexus_test.py b/tests/nexus_test.py index 8a8d59b3..e612eff8 100644 --- a/tests/nexus_test.py +++ b/tests/nexus_test.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) -import uuid +from contextlib import contextmanager from io import BytesIO from pathlib import Path from typing import Union @@ -120,26 +120,29 @@ def _write_nexus_data(store: Union[Path, BytesIO]) -> None: sample.create_field('type', sample_data['type']) -def _file_store(typ): - if typ == Path: - return Path('nexus_test', uuid.uuid4().hex, 'testfile.nxs') - return BytesIO() +@contextmanager +def _file_store(request: pytest.FixtureRequest): + if request.param == BytesIO: + yield BytesIO() + else: + # It would be good to use pyfakefs here, but h5py + # uses C to open files and that bypasses the fake. + base = request.getfixturevalue('tmp_path') + yield base / 'testfile.nxs' @pytest.fixture(params=[Path, BytesIO, snx.Group]) -def nexus_file(fs, request): - _ = fs # Request fs to prevent writing to real disk when param=Path - - store = _file_store(request.param) - _write_nexus_data(store) - if isinstance(store, BytesIO): - store.seek(0) - - if request.param in (Path, BytesIO): - yield store - else: - with snx.File(store, 'r') as f: - yield f +def nexus_file(request): + with _file_store(request) as store: + _write_nexus_data(store) + if isinstance(store, BytesIO): + store.seek(0) + + if request.param in (Path, BytesIO): + yield store + else: + with snx.File(store, 'r') as f: + yield f @pytest.fixture() From 43916397aa10660b6cc11312f10fd2f38cdf4a62 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 6 Mar 2024 16:03:02 +0100 Subject: [PATCH 17/25] Test with multiple entries --- tests/nexus_test.py | 42 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/tests/nexus_test.py b/tests/nexus_test.py index e612eff8..140de13b 100644 --- a/tests/nexus_test.py +++ b/tests/nexus_test.py @@ -82,7 +82,7 @@ def _write_transformation(group: snx.Group, offset: sc.Variable) -> None: def _write_nexus_data(store: Union[Path, BytesIO]) -> None: with snx.File(store, 'w') as root: - entry = root.create_class('entry', snx.NXentry) + entry = root.create_class('entry-001', snx.NXentry) instrument = entry.create_class('reducer', snx.NXinstrument) detector = instrument.create_class('bank12', snx.NXdetector) @@ -197,7 +197,7 @@ def expected_sample() -> sc.DataGroup: return _sample_data() -@pytest.mark.parametrize('entry_name', (None, nexus.EntryName('entry'))) +@pytest.mark.parametrize('entry_name', (None, nexus.EntryName('entry-001'))) def test_load_detector(nexus_file, expected_bank12, entry_name): detector = nexus.load_detector( nexus_file, @@ -207,7 +207,39 @@ def test_load_detector(nexus_file, expected_bank12, entry_name): sc.testing.assert_identical(detector['bank12_events'], expected_bank12) -@pytest.mark.parametrize('entry_name', (None, nexus.EntryName('entry'))) +def test_load_detector_requires_entry_name_if_not_unique(nexus_file): + if not isinstance(nexus_file, Path): + # For simplicity, only create a second entry in an actual file + return + + with snx.File(nexus_file, 'r+') as f: + f.create_class('entry', snx.NXentry) + + with pytest.raises(ValueError): + nexus.load_detector( + nexus.FilePath(nexus_file), + detector_name=nexus.DetectorName('bank12'), + entry_name=None, + ) + + +def test_load_detector_select_entry_if_not_unique(nexus_file, expected_bank12): + if not isinstance(nexus_file, Path): + # For simplicity, only create a second entry in an actual file + return + + with snx.File(nexus_file, 'r+') as f: + f.create_class('entry', snx.NXentry) + + detector = nexus.load_detector( + nexus.FilePath(nexus_file), + detector_name=nexus.DetectorName('bank12'), + entry_name=nexus.EntryName('entry-001'), + ) + sc.testing.assert_identical(detector['bank12_events'], expected_bank12) + + +@pytest.mark.parametrize('entry_name', (None, nexus.EntryName('entry-001'))) def test_load_monitor(nexus_file, expected_monitor, entry_name): monitor = nexus.load_monitor( nexus_file, @@ -217,7 +249,7 @@ def test_load_monitor(nexus_file, expected_monitor, entry_name): sc.testing.assert_identical(monitor['data'], expected_monitor) -@pytest.mark.parametrize('entry_name', (None, nexus.EntryName('entry'))) +@pytest.mark.parametrize('entry_name', (None, nexus.EntryName('entry-001'))) @pytest.mark.parametrize('source_name', (None, nexus.SourceName('source'))) def test_load_source(nexus_file, expected_source, entry_name, source_name): source = nexus.load_source( @@ -231,7 +263,7 @@ def test_load_source(nexus_file, expected_source, entry_name, source_name): sc.testing.assert_identical(source, nexus.RawSource(expected_source)) -@pytest.mark.parametrize('entry_name', (None, nexus.EntryName('entry'))) +@pytest.mark.parametrize('entry_name', (None, nexus.EntryName('entry-001'))) def test_load_sample(nexus_file, expected_sample, entry_name): sample = nexus.load_sample(nexus_file, entry_name=entry_name) sc.testing.assert_identical(sample, nexus.RawSample(expected_sample)) From 1ffdd36620b869a9a5886d32eb536357e7f48c0a Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 6 Mar 2024 16:12:27 +0100 Subject: [PATCH 18/25] Extract data by type not name --- src/ess/reduce/nexus.py | 63 ++++++++++++++++++++++++----------------- tests/nexus_test.py | 47 ++++++++++++++++++++++-------- 2 files changed, 72 insertions(+), 38 deletions(-) diff --git a/src/ess/reduce/nexus.py b/src/ess/reduce/nexus.py index b410f398..bc01fff2 100644 --- a/src/ess/reduce/nexus.py +++ b/src/ess/reduce/nexus.py @@ -268,67 +268,78 @@ def _unique_child_group( return next(iter(children.values())) # type: ignore[return-value] -def extract_detector_data( - detector: RawDetector, detector_name: DetectorName -) -> RawDetectorData: +def extract_detector_data(detector: RawDetector) -> RawDetectorData: """Get and return the events or histogram from a detector loaded from NeXus. + This function looks for a data array in the detector group and returns that. + Parameters ---------- detector: A detector loaded from NeXus. - detector_name: - Name of the detector. Returns ------- : A data array containing the events or histogram. + Raises + ------ + ValueError + If there is more than one data array. + See also -------- load_detector: Load a detector from a NeXus file in a format compatible with ``extract_detector_data``. """ - return RawDetectorData(_extract_events_or_histogram(detector, detector_name)) + return RawDetectorData(_extract_events_or_histogram(detector)) -def extract_monitor_data( - monitor: RawMonitor, monitor_name: MonitorName -) -> RawMonitorData: +def extract_monitor_data(monitor: RawMonitor) -> RawMonitorData: """Get and return the events or histogram from a monitor loaded from NeXus. + This function looks for a data array in the monitor group and returns that. + Parameters ---------- monitor: A monitor loaded from NeXus. - monitor_name: - Name of the monitor. Returns ------- : A data array containing the events or histogram. + Raises + ------ + ValueError + If there is more than one data array. + See also -------- load_monitor: Load a monitor from a NeXus file in a format compatible with ``extract_monitor_data``. """ - return RawMonitorData(_extract_events_or_histogram(monitor, monitor_name)) - - -def _extract_events_or_histogram(dg: sc.DataGroup, name: str) -> sc.DataArray: - data_names = {f'{name}_events', 'data'} - for data_name in data_names: - try: - return dg[data_name] - except KeyError: - pass - raise ValueError( - f"Raw data '{name}' loaded from NeXus does not contain events or a histogram. " - f"Expected to find one of {data_names}, " - f"but the data only contains {set(dg.keys())}" - ) + return RawMonitorData(_extract_events_or_histogram(monitor)) + + +def _extract_events_or_histogram(dg: sc.DataGroup) -> sc.DataArray: + data_arrays = { + key: value for key, value in dg.items() if isinstance(value, sc.DataArray) + } + if len(data_arrays) == 0: + raise ValueError( + "Raw data loaded from NeXus does not contain events or a histogram. " + "Expected to find a data array, " + f"but the data only contains {set(dg.keys())}" + ) + if len(data_arrays) > 1: + raise ValueError( + "Raw data loaded from NeXus contains more than one data array. " + "Cannot uniquely identify the event or histogram data. " + f"Got items {set(dg.keys())}" + ) + return next(iter(data_arrays.values())) diff --git a/tests/nexus_test.py b/tests/nexus_test.py index 140de13b..7eb5a4a5 100644 --- a/tests/nexus_test.py +++ b/tests/nexus_test.py @@ -269,15 +269,38 @@ def test_load_sample(nexus_file, expected_sample, entry_name): sc.testing.assert_identical(sample, nexus.RawSample(expected_sample)) -def test_extract_detector_data(nexus_file, expected_bank12): - detector_name = nexus.DetectorName('bank12') - detector = nexus.load_detector(nexus_file, detector_name=detector_name) - data = nexus.extract_detector_data(detector, detector_name=detector_name) - sc.testing.assert_identical(data, nexus.RawDetectorData(expected_bank12)) - - -def test_extract_monitor_data(nexus_file, expected_monitor): - monitor_name = nexus.MonitorName('monitor') - monitor = nexus.load_monitor(nexus_file, monitor_name=monitor_name) - data = nexus.extract_monitor_data(monitor, monitor_name=monitor_name) - sc.testing.assert_identical(data, nexus.RawMonitorData(expected_monitor)) +def test_extract_detector_data(): + detector = sc.DataGroup( + { + 'jdl2ab': sc.DataArray(sc.arange('xx', 10)), + 'llk': 23, + ' _': sc.linspace('xx', 2, 3, 10), + } + ) + data = nexus.extract_detector_data(nexus.RawDetector(detector)) + sc.testing.assert_identical(data, nexus.RawDetectorData(detector['jdl2ab'])) + + +def test_extract_monitor_data(): + monitor = sc.DataGroup( + { + '(eed)': sc.DataArray(sc.arange('xx', 10)), + 'llk': 23, + ' _': sc.linspace('xx', 2, 3, 10), + } + ) + data = nexus.extract_detector_data(nexus.RawMonitor(monitor)) + sc.testing.assert_identical(data, nexus.RawDetectorData(monitor['(eed)'])) + + +def test_extract_detector_data_requires_unique_data_array(): + detector = sc.DataGroup( + { + 'jdl2ab': sc.DataArray(sc.arange('xx', 10)), + 'llk': 23, + 'lob': sc.DataArray(sc.arange('yy', 20)), + ' _': sc.linspace('xx', 2, 3, 10), + } + ) + with pytest.raises(ValueError): + nexus.extract_detector_data(nexus.RawDetector(detector)) From 4312a8bd6dec1ad61aa50344304548c1c8e5be15 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Thu, 7 Mar 2024 11:51:34 +0100 Subject: [PATCH 19/25] Store combined transformation --- src/ess/reduce/nexus.py | 18 ++++++++++++++---- tests/nexus_test.py | 13 ++++++++++--- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/ess/reduce/nexus.py b/src/ess/reduce/nexus.py index bc01fff2..bb62848f 100644 --- a/src/ess/reduce/nexus.py +++ b/src/ess/reduce/nexus.py @@ -62,7 +62,8 @@ def load_detector( ) -> RawDetector: """Load a single detector (bank) from a NeXus file. - The detector positions are computed automatically for NeXus transformations. + The detector positions are computed automatically from NeXus transformations, + and the combined transformation is stored under the name 'transformation'. Parameters ---------- @@ -105,7 +106,8 @@ def load_monitor( ) -> RawMonitor: """Load a single monitor from a NeXus file. - The monitor position is computed automatically for NeXus transformations. + The monitor position is computed automatically from NeXus transformations, + and the combined transformation is stored under the name 'transformation'. Parameters ---------- @@ -148,7 +150,8 @@ def load_source( ) -> RawSource: """Load a source from a NeXus file. - The source position is computed automatically for NeXus transformations. + The source position is computed automatically from NeXus transformations, + and the combined transformation is stored under the name 'transformation'. Parameters ---------- @@ -234,7 +237,14 @@ def _load_group_with_positions( loaded = cast( sc.DataGroup, _unique_child_group(instrument, nx_class, group_name)[()] ) - loaded = snx.compute_positions(loaded) + + transform_out_name = 'transformation' + if transform_out_name in loaded: + raise RuntimeError( + f"Loaded data contains an item '{transform_out_name}' but we want to " + "store the combined NeXus transformations under that name.") + + loaded = snx.compute_positions(loaded, store_transform=transform_out_name) return loaded diff --git a/tests/nexus_test.py b/tests/nexus_test.py index 7eb5a4a5..8e619476 100644 --- a/tests/nexus_test.py +++ b/tests/nexus_test.py @@ -28,7 +28,6 @@ def _event_data_components() -> sc.DataGroup: dims=['event_index'], unit=None, values=[0, 3, 3, 6] ), 'detector_number': sc.arange('detector_number', 5, unit=None), - 'offset': sc.vector([0.4, 0.0, 11.5], unit='m'), 'pixel_offset': sc.vectors( dims=['detector_number'], values=np.arange(3 * 5).reshape((5, 3)), @@ -38,6 +37,10 @@ def _event_data_components() -> sc.DataGroup: ) +def detector_transformation_components() -> sc.DataGroup: + return sc.DataGroup({ + 'offset': sc.vector([0.4, 0.0, 11.5], unit='m'),}) + def _monitor_histogram() -> sc.DataArray: return sc.DataArray( sc.array(dims=['time'], values=[2, 4, 8, 3], unit='counts'), @@ -55,6 +58,7 @@ def _source_data() -> sc.DataGroup: 'probe': 'neutron', 'type': 'Spallation Neutron Source', 'position': sc.vector([0, 0, 0], unit='m'), + 'transformation': sc.spatial.translation(value=[0, 0, 0], unit='m') } ) @@ -96,7 +100,7 @@ def _write_nexus_data(store: Union[Path, BytesIO]) -> None: detector['y_pixel_offset'] = detector_components['pixel_offset'].fields.y detector['z_pixel_offset'] = detector_components['pixel_offset'].fields.z detector['detector_number'] = detector_components['detector_number'] - _write_transformation(detector, detector_components['offset']) + _write_transformation(detector, detector_transformation_components()['offset']) monitor_data = _monitor_histogram() monitor = instrument.create_class('monitor', snx.NXmonitor) @@ -178,7 +182,8 @@ def expected_bank12(): binned.coords['y_pixel_offset'] = components['pixel_offset'].fields.y binned.coords['z_pixel_offset'] = components['pixel_offset'].fields.z # Computed position - binned.coords['position'] = components['offset'] + components['pixel_offset'] + offset = detector_transformation_components()['offset'] + binned.coords['position'] = offset + components['pixel_offset'] return binned @@ -205,6 +210,8 @@ def test_load_detector(nexus_file, expected_bank12, entry_name): entry_name=entry_name, ) sc.testing.assert_identical(detector['bank12_events'], expected_bank12) + offset = detector_transformation_components()['offset'] + sc.testing.assert_identical(detector['transformation'], sc.spatial.translation(unit=offset.unit, value=offset.value)) def test_load_detector_requires_entry_name_if_not_unique(nexus_file): From dc0d258a720526120a3066c0995d0399aa388721 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Thu, 7 Mar 2024 12:55:53 +0100 Subject: [PATCH 20/25] Add logging module --- src/ess/reduce/logging.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 src/ess/reduce/logging.py diff --git a/src/ess/reduce/logging.py b/src/ess/reduce/logging.py new file mode 100644 index 00000000..05adcfaf --- /dev/null +++ b/src/ess/reduce/logging.py @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) + +"""Logging tools for ess.reduce.""" + +import logging + + +def get_logger() -> logging.Logger: + """Return the logger for ess.reduce. + + Returns + ------- + : + The requested logger. + """ + return logging.getLogger('scipp.ess.reduce') From 1a388a82a95252a8419ea75ecda82d69ac0ffa1b Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Thu, 7 Mar 2024 12:56:06 +0100 Subject: [PATCH 21/25] Extract data by type not name --- src/ess/reduce/nexus.py | 56 ++++++++++++++++++++++++++++++----------- tests/nexus_test.py | 55 ++++++++++++++++++++++++++++++++-------- 2 files changed, 86 insertions(+), 25 deletions(-) diff --git a/src/ess/reduce/nexus.py b/src/ess/reduce/nexus.py index bb62848f..4e9851d4 100644 --- a/src/ess/reduce/nexus.py +++ b/src/ess/reduce/nexus.py @@ -17,6 +17,8 @@ import scipp as sc import scippnexus as snx +from .logging import get_logger + FilePath = NewType('FilePath', Path) """Full path to a NeXus file on disk.""" NeXusFile = NewType('NeXusFile', BinaryIO) @@ -242,7 +244,8 @@ def _load_group_with_positions( if transform_out_name in loaded: raise RuntimeError( f"Loaded data contains an item '{transform_out_name}' but we want to " - "store the combined NeXus transformations under that name.") + "store the combined NeXus transformations under that name." + ) loaded = snx.compute_positions(loaded, store_transform=transform_out_name) return loaded @@ -337,19 +340,44 @@ def extract_monitor_data(monitor: RawMonitor) -> RawMonitorData: def _extract_events_or_histogram(dg: sc.DataGroup) -> sc.DataArray: - data_arrays = { - key: value for key, value in dg.items() if isinstance(value, sc.DataArray) + event_data_arrays = { + key: value + for key, value in dg.items() + if isinstance(value, sc.DataArray) and value.bins is not None } - if len(data_arrays) == 0: - raise ValueError( - "Raw data loaded from NeXus does not contain events or a histogram. " - "Expected to find a data array, " - f"but the data only contains {set(dg.keys())}" - ) - if len(data_arrays) > 1: + histogram_data_arrays = { + key: value + for key, value in dg.items() + if isinstance(value, sc.DataArray) and value.bins is None + } + if (array := _select_unique_array(event_data_arrays, 'event')) is not None: + if histogram_data_arrays: + get_logger().info( + "Selecting event data '%s' in favor of histogram data {%s}", + next(iter(event_data_arrays.keys())), + ', '.join(map(lambda k: f"'{k}'", histogram_data_arrays)), + ) + return array + + if (array := _select_unique_array(histogram_data_arrays, 'histogram')) is not None: + return array + + raise ValueError( + "Raw data loaded from NeXus does not contain events or a histogram. " + "Expected to find a data array, " + f"but the data only contains {set(dg.keys())}" + ) + + +def _select_unique_array( + arrays: dict[str, sc.DataArray], mapping_name: str +) -> Optional[sc.DataArray]: + if not arrays: + return None + if len(arrays) > 1: raise ValueError( - "Raw data loaded from NeXus contains more than one data array. " - "Cannot uniquely identify the event or histogram data. " - f"Got items {set(dg.keys())}" + f"Raw data loaded from NeXus contains more than one {mapping_name} " + "data array. Cannot uniquely identify the data to extract. " + f"Got {mapping_name} items {set(arrays.keys())}" ) - return next(iter(data_arrays.values())) + return next(iter(arrays.values())) diff --git a/tests/nexus_test.py b/tests/nexus_test.py index 8e619476..0c207ebd 100644 --- a/tests/nexus_test.py +++ b/tests/nexus_test.py @@ -38,8 +38,12 @@ def _event_data_components() -> sc.DataGroup: def detector_transformation_components() -> sc.DataGroup: - return sc.DataGroup({ - 'offset': sc.vector([0.4, 0.0, 11.5], unit='m'),}) + return sc.DataGroup( + { + 'offset': sc.vector([0.4, 0.0, 11.5], unit='m'), + } + ) + def _monitor_histogram() -> sc.DataArray: return sc.DataArray( @@ -58,7 +62,7 @@ def _source_data() -> sc.DataGroup: 'probe': 'neutron', 'type': 'Spallation Neutron Source', 'position': sc.vector([0, 0, 0], unit='m'), - 'transformation': sc.spatial.translation(value=[0, 0, 0], unit='m') + 'transformation': sc.spatial.translation(value=[0, 0, 0], unit='m'), } ) @@ -211,7 +215,10 @@ def test_load_detector(nexus_file, expected_bank12, entry_name): ) sc.testing.assert_identical(detector['bank12_events'], expected_bank12) offset = detector_transformation_components()['offset'] - sc.testing.assert_identical(detector['transformation'], sc.spatial.translation(unit=offset.unit, value=offset.value)) + sc.testing.assert_identical( + detector['transformation'], + sc.spatial.translation(unit=offset.unit, value=offset.value), + ) def test_load_detector_requires_entry_name_if_not_unique(nexus_file): @@ -279,7 +286,7 @@ def test_load_sample(nexus_file, expected_sample, entry_name): def test_extract_detector_data(): detector = sc.DataGroup( { - 'jdl2ab': sc.DataArray(sc.arange('xx', 10)), + 'jdl2ab': sc.data.binned_x(10, 3), 'llk': 23, ' _': sc.linspace('xx', 2, 3, 10), } @@ -291,23 +298,49 @@ def test_extract_detector_data(): def test_extract_monitor_data(): monitor = sc.DataGroup( { - '(eed)': sc.DataArray(sc.arange('xx', 10)), + '(eed)': sc.data.data_xy(), 'llk': 23, ' _': sc.linspace('xx', 2, 3, 10), } ) - data = nexus.extract_detector_data(nexus.RawMonitor(monitor)) - sc.testing.assert_identical(data, nexus.RawDetectorData(monitor['(eed)'])) + data = nexus.extract_monitor_data(nexus.RawMonitor(monitor)) + sc.testing.assert_identical(data, nexus.RawMonitorData(monitor['(eed)'])) -def test_extract_detector_data_requires_unique_data_array(): +def test_extract_detector_data_requires_unique_dense_data(): detector = sc.DataGroup( { - 'jdl2ab': sc.DataArray(sc.arange('xx', 10)), + 'jdl2ab': sc.data.data_xy(), 'llk': 23, - 'lob': sc.DataArray(sc.arange('yy', 20)), + 'lob': sc.data.data_xy(), ' _': sc.linspace('xx', 2, 3, 10), } ) with pytest.raises(ValueError): nexus.extract_detector_data(nexus.RawDetector(detector)) + + +def test_extract_detector_data_requires_unique_event_data(): + detector = sc.DataGroup( + { + 'jdl2ab': sc.data.binned_x(10, 3), + 'llk': 23, + 'lob': sc.data.binned_x(14, 5), + ' _': sc.linspace('xx', 2, 3, 10), + } + ) + with pytest.raises(ValueError): + nexus.extract_detector_data(nexus.RawDetector(detector)) + + +def test_extract_detector_data_favors_event_data_over_histogram_data(): + detector = sc.DataGroup( + { + 'jdl2ab': sc.data.data_xy(), + 'llk': 23, + 'lob': sc.data.binned_x(14, 5), + ' _': sc.linspace('xx', 2, 3, 10), + } + ) + data = nexus.extract_detector_data(nexus.RawDetector(detector)) + sc.testing.assert_identical(data, nexus.RawDetectorData(detector['lob'])) From 52c2eae5ae244c5f731ed20e7dfb20ce831ebd2f Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Thu, 7 Mar 2024 15:10:44 +0100 Subject: [PATCH 22/25] Fix typo --- src/ess/reduce/nexus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ess/reduce/nexus.py b/src/ess/reduce/nexus.py index 4e9851d4..600b9d9d 100644 --- a/src/ess/reduce/nexus.py +++ b/src/ess/reduce/nexus.py @@ -5,7 +5,7 @@ This module defines functions and domain types that can be used to build Sciline pipelines for simple workflows. -If multiple different kind sof files (e.g., sample and background runs) +If multiple different kinds of files (e.g., sample and background runs) are needed, custom types and providers need to be defined to wrap the basic ones here. """ From 4446a0046efd17b2d707491d1e9f406d555dc049 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Thu, 7 Mar 2024 15:12:22 +0100 Subject: [PATCH 23/25] Check for name conflict with position --- src/ess/reduce/nexus.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/ess/reduce/nexus.py b/src/ess/reduce/nexus.py index 600b9d9d..c356d065 100644 --- a/src/ess/reduce/nexus.py +++ b/src/ess/reduce/nexus.py @@ -246,8 +246,16 @@ def _load_group_with_positions( f"Loaded data contains an item '{transform_out_name}' but we want to " "store the combined NeXus transformations under that name." ) + position_out_name = 'position' + if position_out_name in loaded: + raise RuntimeError( + f"Loaded data contains an item '{position_out_name}' but we want to " + "store the computed positions under that name." + ) - loaded = snx.compute_positions(loaded, store_transform=transform_out_name) + loaded = snx.compute_positions( + loaded, store_position=position_out_name, store_transform=transform_out_name + ) return loaded From 3a51a10d1ccd95c0199e439b6ee7db4d93f15118 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Fri, 8 Mar 2024 10:34:25 +0100 Subject: [PATCH 24/25] Add NeXus prefix to Names --- src/ess/reduce/nexus.py | 24 ++++++++++++------------ tests/nexus_test.py | 20 ++++++++++---------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/ess/reduce/nexus.py b/src/ess/reduce/nexus.py index c356d065..371eecb1 100644 --- a/src/ess/reduce/nexus.py +++ b/src/ess/reduce/nexus.py @@ -33,13 +33,13 @@ NeXusGroup = NewType('NeXusGroup', snx.Group) """A ScippNexus group in an open file.""" -DetectorName = NewType('DetectorName', str) +NeXusDetectorName = NewType('NeXusDetectorName', str) """Name of a detector (bank) in a NeXus file.""" -EntryName = NewType('EntryName', str) +NeXusEntryName = NewType('NeXusEntryName', str) """Name of an entry in a NeXus file.""" -MonitorName = NewType('MonitorName', str) +NeXusMonitorName = NewType('NeXusMonitorName', str) """Name of a monitor in a NeXus file.""" -SourceName = NewType('SourceName', str) +NeXusSourceName = NewType('NeXusSourceName', str) """Name of a source in a NeXus file.""" RawDetector = NewType('RawDetector', sc.DataGroup) @@ -59,8 +59,8 @@ def load_detector( file_path: Union[FilePath, NeXusFile, NeXusGroup], *, - detector_name: DetectorName, - entry_name: Optional[EntryName] = None, + detector_name: NeXusDetectorName, + entry_name: Optional[NeXusEntryName] = None, ) -> RawDetector: """Load a single detector (bank) from a NeXus file. @@ -103,8 +103,8 @@ def load_detector( def load_monitor( file_path: Union[FilePath, NeXusFile, NeXusGroup], *, - monitor_name: MonitorName, - entry_name: Optional[EntryName] = None, + monitor_name: NeXusMonitorName, + entry_name: Optional[NeXusEntryName] = None, ) -> RawMonitor: """Load a single monitor from a NeXus file. @@ -147,8 +147,8 @@ def load_monitor( def load_source( file_path: Union[FilePath, NeXusFile, NeXusGroup], *, - source_name: Optional[SourceName] = None, - entry_name: Optional[EntryName] = None, + source_name: Optional[NeXusSourceName] = None, + entry_name: Optional[NeXusEntryName] = None, ) -> RawSource: """Load a source from a NeXus file. @@ -192,7 +192,7 @@ def load_source( def load_sample( file_path: Union[FilePath, NeXusFile, NeXusGroup], - entry_name: Optional[EntryName] = None, + entry_name: Optional[NeXusEntryName] = None, ) -> RawSample: """Load a sample from a NeXus file. @@ -231,7 +231,7 @@ def _load_group_with_positions( *, group_name: Optional[str], nx_class: Type[snx.NXobject], - entry_name: Optional[EntryName] = None, + entry_name: Optional[NeXusEntryName] = None, ) -> sc.DataGroup: with _open_nexus_file(file_path) as f: entry = _unique_child_group(f, snx.NXentry, entry_name) diff --git a/tests/nexus_test.py b/tests/nexus_test.py index 0c207ebd..1570ebf9 100644 --- a/tests/nexus_test.py +++ b/tests/nexus_test.py @@ -206,11 +206,11 @@ def expected_sample() -> sc.DataGroup: return _sample_data() -@pytest.mark.parametrize('entry_name', (None, nexus.EntryName('entry-001'))) +@pytest.mark.parametrize('entry_name', (None, nexus.NeXusEntryName('entry-001'))) def test_load_detector(nexus_file, expected_bank12, entry_name): detector = nexus.load_detector( nexus_file, - detector_name=nexus.DetectorName('bank12'), + detector_name=nexus.NeXusDetectorName('bank12'), entry_name=entry_name, ) sc.testing.assert_identical(detector['bank12_events'], expected_bank12) @@ -232,7 +232,7 @@ def test_load_detector_requires_entry_name_if_not_unique(nexus_file): with pytest.raises(ValueError): nexus.load_detector( nexus.FilePath(nexus_file), - detector_name=nexus.DetectorName('bank12'), + detector_name=nexus.NeXusDetectorName('bank12'), entry_name=None, ) @@ -247,24 +247,24 @@ def test_load_detector_select_entry_if_not_unique(nexus_file, expected_bank12): detector = nexus.load_detector( nexus.FilePath(nexus_file), - detector_name=nexus.DetectorName('bank12'), - entry_name=nexus.EntryName('entry-001'), + detector_name=nexus.NeXusDetectorName('bank12'), + entry_name=nexus.NeXusEntryName('entry-001'), ) sc.testing.assert_identical(detector['bank12_events'], expected_bank12) -@pytest.mark.parametrize('entry_name', (None, nexus.EntryName('entry-001'))) +@pytest.mark.parametrize('entry_name', (None, nexus.NeXusEntryName('entry-001'))) def test_load_monitor(nexus_file, expected_monitor, entry_name): monitor = nexus.load_monitor( nexus_file, - monitor_name=nexus.MonitorName('monitor'), + monitor_name=nexus.NeXusMonitorName('monitor'), entry_name=entry_name, ) sc.testing.assert_identical(monitor['data'], expected_monitor) -@pytest.mark.parametrize('entry_name', (None, nexus.EntryName('entry-001'))) -@pytest.mark.parametrize('source_name', (None, nexus.SourceName('source'))) +@pytest.mark.parametrize('entry_name', (None, nexus.NeXusEntryName('entry-001'))) +@pytest.mark.parametrize('source_name', (None, nexus.NeXusSourceName('source'))) def test_load_source(nexus_file, expected_source, entry_name, source_name): source = nexus.load_source( nexus_file, @@ -277,7 +277,7 @@ def test_load_source(nexus_file, expected_source, entry_name, source_name): sc.testing.assert_identical(source, nexus.RawSource(expected_source)) -@pytest.mark.parametrize('entry_name', (None, nexus.EntryName('entry-001'))) +@pytest.mark.parametrize('entry_name', (None, nexus.NeXusEntryName('entry-001'))) def test_load_sample(nexus_file, expected_sample, entry_name): sample = nexus.load_sample(nexus_file, entry_name=entry_name) sc.testing.assert_identical(sample, nexus.RawSample(expected_sample)) From 99a6a4837487dbcafec7bb1d19e682060333a69c Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Mon, 11 Mar 2024 09:03:36 +0100 Subject: [PATCH 25/25] Rename transformation -> transform --- src/ess/reduce/nexus.py | 8 ++++---- tests/nexus_test.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ess/reduce/nexus.py b/src/ess/reduce/nexus.py index 371eecb1..54e3eb1e 100644 --- a/src/ess/reduce/nexus.py +++ b/src/ess/reduce/nexus.py @@ -65,7 +65,7 @@ def load_detector( """Load a single detector (bank) from a NeXus file. The detector positions are computed automatically from NeXus transformations, - and the combined transformation is stored under the name 'transformation'. + and the combined transformation is stored under the name 'transform'. Parameters ---------- @@ -109,7 +109,7 @@ def load_monitor( """Load a single monitor from a NeXus file. The monitor position is computed automatically from NeXus transformations, - and the combined transformation is stored under the name 'transformation'. + and the combined transformation is stored under the name 'transform'. Parameters ---------- @@ -153,7 +153,7 @@ def load_source( """Load a source from a NeXus file. The source position is computed automatically from NeXus transformations, - and the combined transformation is stored under the name 'transformation'. + and the combined transformation is stored under the name 'transform'. Parameters ---------- @@ -240,7 +240,7 @@ def _load_group_with_positions( sc.DataGroup, _unique_child_group(instrument, nx_class, group_name)[()] ) - transform_out_name = 'transformation' + transform_out_name = 'transform' if transform_out_name in loaded: raise RuntimeError( f"Loaded data contains an item '{transform_out_name}' but we want to " diff --git a/tests/nexus_test.py b/tests/nexus_test.py index 1570ebf9..6287c106 100644 --- a/tests/nexus_test.py +++ b/tests/nexus_test.py @@ -62,7 +62,7 @@ def _source_data() -> sc.DataGroup: 'probe': 'neutron', 'type': 'Spallation Neutron Source', 'position': sc.vector([0, 0, 0], unit='m'), - 'transformation': sc.spatial.translation(value=[0, 0, 0], unit='m'), + 'transform': sc.spatial.translation(value=[0, 0, 0], unit='m'), } ) @@ -216,7 +216,7 @@ def test_load_detector(nexus_file, expected_bank12, entry_name): sc.testing.assert_identical(detector['bank12_events'], expected_bank12) offset = detector_transformation_components()['offset'] sc.testing.assert_identical( - detector['transformation'], + detector['transform'], sc.spatial.translation(unit=offset.unit, value=offset.value), )