From 54aca77de9b7247848eb60c8b2f64fd34098ab15 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Mon, 18 Nov 2024 13:51:11 +0100 Subject: [PATCH 1/5] Depend on scippneutron --- .copier-answers.yml | 2 +- pyproject.toml | 1 + requirements/base.in | 1 + requirements/base.txt | 49 +++++++++++++-- requirements/basetest.txt | 6 +- requirements/ci.txt | 4 +- requirements/dev.txt | 12 ++-- requirements/docs.txt | 13 ++-- requirements/nightly.in | 5 ++ requirements/nightly.txt | 125 ++++++++++++++++++++++++++++++++++++-- requirements/static.txt | 2 +- requirements/wheels.txt | 4 +- tox.ini | 2 +- 13 files changed, 189 insertions(+), 37 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 267a33a7..7623a748 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -5,7 +5,7 @@ description: Common data reduction tools for the ESS facility max_python: '3.13' min_python: '3.10' namespace_package: ess -nightly_deps: scippnexus,scipp,sciline,cyclebane +nightly_deps: scippnexus,scipp,sciline,cyclebane,scippneutron orgname: scipp prettyname: ESSreduce projectname: essreduce diff --git a/pyproject.toml b/pyproject.toml index 2bd025a6..51d1935b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ requires-python = ">=3.10" dependencies = [ "sciline>=24.06.2", "scipp>=24.02.0", + "scippneutron>=24.11.0", "scippnexus>=24.11.0", ] diff --git a/requirements/base.in b/requirements/base.in index f57c96d3..7a89b7c5 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -4,4 +4,5 @@ # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! sciline>=24.06.2 scipp>=24.02.0 +scippneutron>=24.11.0 scippnexus>=24.11.0 diff --git a/requirements/base.txt b/requirements/base.txt index 6e82eeb6..d504ab93 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,32 +1,69 @@ -# SHA1:61ced0b7e1301f66609a2d72adaad2ad2e729000 +# SHA1:41ad412be1c3ef8654797c35e9ebb78554531f85 # # This file is autogenerated by pip-compile-multi # To update, run: # # pip-compile-multi # +contourpy==1.3.1 + # via matplotlib cyclebane==24.10.0 # via sciline +cycler==0.12.1 + # via matplotlib +fonttools==4.55.0 + # via matplotlib h5py==3.12.1 - # via scippnexus + # via + # scippneutron + # scippnexus +kiwisolver==1.4.7 + # via matplotlib +matplotlib==3.9.2 + # via + # mpltoolbox + # plopp +mpltoolbox==24.5.1 + # via scippneutron networkx==3.4.2 # via cyclebane numpy==2.1.3 # via + # contourpy # h5py + # matplotlib + # mpltoolbox # scipp + # scippneutron # scipy +packaging==24.2 + # via matplotlib +pillow==11.0.0 + # via matplotlib +plopp==24.10.0 + # via scippneutron +pyparsing==3.2.0 + # via matplotlib python-dateutil==2.9.0.post0 - # via scippnexus -sciline==24.6.3 + # via + # matplotlib + # scippnexus +sciline==24.10.0 # via -r base.in scipp==24.11.1 # via # -r base.in + # scippneutron # scippnexus -scippnexus==24.11.0 +scippneutron==24.11.0 # via -r base.in +scippnexus==24.11.0 + # via + # -r base.in + # scippneutron scipy==1.14.1 - # via scippnexus + # via + # scippneutron + # scippnexus six==1.16.0 # via python-dateutil diff --git a/requirements/basetest.txt b/requirements/basetest.txt index dea9ccc5..f189f102 100644 --- a/requirements/basetest.txt +++ b/requirements/basetest.txt @@ -29,13 +29,13 @@ ipython==8.29.0 # via ipywidgets ipywidgets==8.1.5 # via -r basetest.in -jedi==0.19.1 +jedi==0.19.2 # via ipython jupyterlab-widgets==3.0.13 # via ipywidgets matplotlib-inline==0.1.7 # via ipython -packaging==24.1 +packaging==24.2 # via # pooch # pytest @@ -65,7 +65,7 @@ six==1.16.0 # via asttokens stack-data==0.6.3 # via ipython -tomli==2.0.2 +tomli==2.1.0 # via pytest traitlets==5.14.3 # via diff --git a/requirements/ci.txt b/requirements/ci.txt index 5d7be1da..14921a29 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -27,7 +27,7 @@ gitpython==3.1.43 # via -r ci.in idna==3.10 # via requests -packaging==24.1 +packaging==24.2 # via # -r ci.in # pyproject-api @@ -44,7 +44,7 @@ requests==2.32.3 # via -r ci.in smmap==5.0.1 # via gitdb -tomli==2.0.2 +tomli==2.1.0 # via # pyproject-api # tox diff --git a/requirements/dev.txt b/requirements/dev.txt index aab445e2..e0ff645a 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -34,7 +34,7 @@ click==8.1.7 # pip-tools copier==9.4.1 # via -r dev.in -dunamai==1.22.0 +dunamai==1.23.0 # via copier fqdn==1.5.1 # via jsonschema @@ -42,7 +42,7 @@ funcy==2.0 # via copier h11==0.14.0 # via httpcore -httpcore==1.0.6 +httpcore==1.0.7 # via httpx httpx==0.27.2 # via jupyterlab @@ -50,7 +50,7 @@ isoduration==20.11.0 # via jsonschema jinja2-ansible-filters==1.3.2 # via copier -json5==0.9.25 +json5==0.9.28 # via jupyterlab-server jsonpointer==3.0.0 # via jsonschema @@ -71,7 +71,7 @@ jupyter-server==2.14.2 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.3.0 +jupyterlab==4.3.1 # via -r dev.in jupyterlab-server==2.27.3 # via jupyterlab @@ -123,11 +123,11 @@ types-python-dateutil==2.9.0.20241003 # via arrow uri-template==1.3.0 # via jsonschema -webcolors==24.8.0 +webcolors==24.11.1 # via jsonschema websocket-client==1.8.0 # via jupyter-server -wheel==0.44.0 +wheel==0.45.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/docs.txt b/requirements/docs.txt index 56e45622..15b223f0 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -34,7 +34,7 @@ comm==0.2.2 # via # ipykernel # ipywidgets -debugpy==1.8.7 +debugpy==1.8.8 # via ipykernel decorator==5.1.1 # via ipython @@ -67,7 +67,7 @@ ipython==8.29.0 # ipywidgets ipywidgets==8.1.5 # via -r docs.in -jedi==0.19.1 +jedi==0.19.2 # via ipython jinja2==3.1.4 # via @@ -127,11 +127,6 @@ nbsphinx==0.9.5 # via -r docs.in nest-asyncio==1.6.0 # via ipykernel -packaging==24.1 - # via - # ipykernel - # nbconvert - # sphinx pandocfilters==1.5.1 # via nbconvert parso==0.8.4 @@ -169,7 +164,7 @@ referencing==0.35.1 # jsonschema-specifications requests==2.32.3 # via sphinx -rpds-py==0.20.1 +rpds-py==0.21.0 # via # jsonschema # referencing @@ -208,7 +203,7 @@ stack-data==0.6.3 # via ipython tinycss2==1.4.0 # via nbconvert -tomli==2.0.2 +tomli==2.1.0 # via sphinx tornado==6.4.1 # via diff --git a/requirements/nightly.in b/requirements/nightly.in index 4256c698..d87643f6 100644 --- a/requirements/nightly.in +++ b/requirements/nightly.in @@ -1,6 +1,11 @@ + # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! +ipywidgets +pooch +pytest scippnexus @ git+https://github.com/scipp/scippnexus@main scipp @ https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl sciline @ git+https://github.com/scipp/sciline@main cyclebane @ git+https://github.com/scipp/cyclebane@main +scippneutron @ git+https://github.com/scipp/scippneutron@main diff --git a/requirements/nightly.txt b/requirements/nightly.txt index 78fdbf23..56fe6f52 100644 --- a/requirements/nightly.txt +++ b/requirements/nightly.txt @@ -1,33 +1,146 @@ -# SHA1:1a4d1263589d9d56c53d9038ebba5e1a66bbd09b +# SHA1:99a76dd6422a3c33a952108f215692b2ca32c80e # # This file is autogenerated by pip-compile-multi # To update, run: # # pip-compile-multi # --r basetest.txt +asttokens==2.4.1 + # via stack-data +certifi==2024.8.30 + # via requests +charset-normalizer==3.4.0 + # via requests +comm==0.2.2 + # via ipywidgets +contourpy==1.3.1 + # via matplotlib cyclebane @ git+https://github.com/scipp/cyclebane@main # via # -r nightly.in # sciline +cycler==0.12.1 + # via matplotlib +decorator==5.1.1 + # via ipython +exceptiongroup==1.2.2 + # via + # ipython + # pytest +executing==2.1.0 + # via stack-data +fonttools==4.55.0 + # via matplotlib h5py==3.12.1 - # via scippnexus + # via + # scippneutron + # scippnexus +idna==3.10 + # via requests +iniconfig==2.0.0 + # via pytest +ipython==8.29.0 + # via ipywidgets +ipywidgets==8.1.5 + # via -r nightly.in +jedi==0.19.2 + # via ipython +jupyterlab-widgets==3.0.13 + # via ipywidgets +kiwisolver==1.4.7 + # via matplotlib +matplotlib==3.9.2 + # via + # mpltoolbox + # plopp +matplotlib-inline==0.1.7 + # via ipython +mpltoolbox==24.5.1 + # via scippneutron networkx==3.4.2 # via cyclebane numpy==2.1.3 # via + # contourpy # h5py + # matplotlib + # mpltoolbox # scipp + # scippneutron # scipy +packaging==24.2 + # via + # matplotlib + # pooch + # pytest +parso==0.8.4 + # via jedi +pexpect==4.9.0 + # via ipython +pillow==11.0.0 + # via matplotlib +platformdirs==4.3.6 + # via pooch +plopp==24.10.0 + # via scippneutron +pluggy==1.5.0 + # via pytest +pooch==1.8.2 + # via -r nightly.in +prompt-toolkit==3.0.48 + # via ipython +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.3 + # via stack-data +pygments==2.18.0 + # via ipython +pyparsing==3.2.0 + # via matplotlib +pytest==8.3.3 + # via -r nightly.in python-dateutil==2.9.0.post0 - # via scippnexus + # via + # matplotlib + # scippnexus +requests==2.32.3 + # via pooch sciline @ git+https://github.com/scipp/sciline@main # via -r nightly.in scipp @ https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl # via # -r nightly.in + # scippneutron # scippnexus -scippnexus @ git+https://github.com/scipp/scippnexus@main +scippneutron @ git+https://github.com/scipp/scippneutron@main # via -r nightly.in +scippnexus @ git+https://github.com/scipp/scippnexus@main + # via + # -r nightly.in + # scippneutron scipy==1.14.1 - # via scippnexus + # via + # scippneutron + # scippnexus +six==1.16.0 + # via + # asttokens + # python-dateutil +stack-data==0.6.3 + # via ipython +tomli==2.1.0 + # via pytest +traitlets==5.14.3 + # via + # comm + # ipython + # ipywidgets + # matplotlib-inline +typing-extensions==4.12.2 + # via ipython +urllib3==2.2.3 + # via requests +wcwidth==0.2.13 + # via prompt-toolkit +widgetsnbextension==4.0.13 + # via ipywidgets diff --git a/requirements/static.txt b/requirements/static.txt index dd21341b..e107d915 100644 --- a/requirements/static.txt +++ b/requirements/static.txt @@ -11,7 +11,7 @@ distlib==0.3.9 # via virtualenv filelock==3.16.1 # via virtualenv -identify==2.6.1 +identify==2.6.2 # via pre-commit nodeenv==1.9.1 # via pre-commit diff --git a/requirements/wheels.txt b/requirements/wheels.txt index d1c1063b..6a1c0600 100644 --- a/requirements/wheels.txt +++ b/requirements/wheels.txt @@ -7,9 +7,9 @@ # build==1.2.2.post1 # via -r wheels.in -packaging==24.1 +packaging==24.2 # via build pyproject-hooks==1.2.0 # via build -tomli==2.0.2 +tomli==2.1.0 # via build diff --git a/tox.ini b/tox.ini index 7d83f0bc..d0253400 100644 --- a/tox.ini +++ b/tox.ini @@ -63,5 +63,5 @@ deps = tomli skip_install = true changedir = requirements -commands = python ./make_base.py --nightly scippnexus,scipp,sciline,cyclebane +commands = python ./make_base.py --nightly scippnexus,scipp,sciline,cyclebane,scippneutron pip-compile-multi -d . --backtracking From 1d4a75c836079a25b6c7eb863859b9582cdd6ed4 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Mon, 18 Nov 2024 15:55:46 +0100 Subject: [PATCH 2/5] Add ability to load all choppers from NeXus --- src/ess/reduce/data.py | 26 ++++++--- src/ess/reduce/nexus/__init__.py | 2 + src/ess/reduce/nexus/_nexus_loader.py | 63 +++++++++++++++++----- src/ess/reduce/nexus/types.py | 20 +++++-- src/ess/reduce/nexus/workflow.py | 78 ++++++++++++++++++++++++++- tests/nexus/nexus_loader_test.py | 45 ++++++++++++++++ tests/nexus/workflow_test.py | 20 +++++++ 7 files changed, 230 insertions(+), 24 deletions(-) diff --git a/src/ess/reduce/data.py b/src/ess/reduce/data.py index ff3e9dc4..63fff011 100644 --- a/src/ess/reduce/data.py +++ b/src/ess/reduce/data.py @@ -31,7 +31,16 @@ def get_path(self, name: str, unzip: bool = False) -> str: return self._registry.fetch(name, processor=pooch.Unzip() if unzip else None) -_registry = Registry( +_bifrost_registry = Registry( + instrument='bifrost', + files={ + "BIFROST_20240914T053723.h5": "md5:0f2fa5c9a851f8e3a4fa61defaa3752e", + }, + version='1', +) + + +_loki_registry = Registry( instrument='loki', files={ # Files from LoKI@Larmor detector test experiment @@ -57,26 +66,31 @@ def get_path(self, name: str, unzip: bool = False) -> str: ) +def bifrost_simulated_elastic() -> str: + """McStas simulation with elastic incoherent scattering + phonon.""" + return _bifrost_registry.get_path('BIFROST_20240914T053723.h5') + + def loki_tutorial_sample_run_60250() -> str: """Sample run with sample and sample holder/can, no transmission monitor in beam.""" - return _registry.get_path('60250-2022-02-28_2215.nxs') + return _loki_registry.get_path('60250-2022-02-28_2215.nxs') def loki_tutorial_sample_run_60339() -> str: """Sample run with sample and sample holder/can, no transmission monitor in beam.""" - return _registry.get_path('60339-2022-02-28_2215.nxs') + return _loki_registry.get_path('60339-2022-02-28_2215.nxs') def loki_tutorial_background_run_60248() -> str: """Background run with sample holder/can only, no transmission monitor.""" - return _registry.get_path('60248-2022-02-28_2215.nxs') + return _loki_registry.get_path('60248-2022-02-28_2215.nxs') def loki_tutorial_background_run_60393() -> str: """Background run with sample holder/can only, no transmission monitor.""" - return _registry.get_path('60393-2022-02-28_2215.nxs') + return _loki_registry.get_path('60393-2022-02-28_2215.nxs') def loki_tutorial_sample_transmission_run() -> str: """Sample transmission run (sample + sample holder/can + transmission monitor).""" - return _registry.get_path('60394-2022-02-28_2215.nxs') + return _loki_registry.get_path('60394-2022-02-28_2215.nxs') diff --git a/src/ess/reduce/nexus/__init__.py b/src/ess/reduce/nexus/__init__.py index 1f52e32f..e666ac76 100644 --- a/src/ess/reduce/nexus/__init__.py +++ b/src/ess/reduce/nexus/__init__.py @@ -18,6 +18,7 @@ load_data, group_event_data, load_component, + load_all_components, compute_component_position, extract_signal_data_array, ) @@ -25,6 +26,7 @@ __all__ = [ 'types', 'group_event_data', + 'load_all_components', 'load_data', 'load_component', 'compute_component_position', diff --git a/src/ess/reduce/nexus/_nexus_loader.py b/src/ess/reduce/nexus/_nexus_loader.py index 35243b0e..47782016 100644 --- a/src/ess/reduce/nexus/_nexus_loader.py +++ b/src/ess/reduce/nexus/_nexus_loader.py @@ -3,8 +3,8 @@ """NeXus loaders.""" -from collections.abc import Mapping -from contextlib import AbstractContextManager, nullcontext +from collections.abc import Generator, Mapping +from contextlib import AbstractContextManager, contextmanager, nullcontext from dataclasses import dataclass from math import prod from typing import cast @@ -28,22 +28,41 @@ def load_component( nx_class: type[snx.NXobject], definitions: Mapping | None | NoNewDefinitionsType = NoNewDefinitions, ) -> sc.DataGroup: - file_path = location.filename + """Load a single component of a given class from NeXus.""" selection = location.selection - entry_name = location.entry_name group_name = location.component_name - with _open_nexus_file(file_path, definitions=definitions) as f: - entry = _unique_child_group(f, snx.NXentry, entry_name) - if nx_class is snx.NXsample: - instrument = entry - else: - instrument = _unique_child_group(entry, snx.NXinstrument, None) - component = _unique_child_group(instrument, nx_class, group_name) + with _open_component_parent( + location, nx_class=nx_class, definitions=definitions + ) as parent: + component = _unique_child_group(parent, nx_class, group_name) loaded = cast(sc.DataGroup, component[selection]) - loaded['nexus_component_name'] = component.name.split('/')[-1] + loaded['nexus_component_name'] = component.name.rsplit('/', 1)[-1] return loaded +def load_all_components( + location: NeXusLocationSpec, + *, + nx_class: type[snx.NXobject], + definitions: Mapping | None | NoNewDefinitionsType = NoNewDefinitions, +) -> sc.DataGroup: + """Load all components of a given class from NeXus.""" + if location.component_name is not None: + raise ValueError( + "`location.component_name` must be None when loading all " + "components of a certain class." + ) + with _open_component_parent( + location, nx_class=nx_class, definitions=definitions + ) as parent: + components = sc.DataGroup() + for name, component in parent[nx_class].items(): + loaded = component[location.selection] + loaded['nexus_component_name'] = name + components[name] = loaded + return components + + def compute_component_position(dg: sc.DataGroup) -> sc.DataGroup: # In some downstream packages we use some of the Nexus components which attempt # to compute positions without having actual Nexus data defining depends_on chains. @@ -70,7 +89,7 @@ def compute_component_position(dg: sc.DataGroup) -> sc.DataGroup: def _open_nexus_file( file_path: FilePath | NeXusFile | NeXusGroup, definitions: Mapping | None | NoNewDefinitionsType = NoNewDefinitions, -) -> AbstractContextManager: +) -> AbstractContextManager[snx.Group]: if isinstance(file_path, getattr(NeXusGroup, '__supertype__', type(None))): if ( definitions is not NoNewDefinitions @@ -85,6 +104,24 @@ def _open_nexus_file( return snx.File(file_path, definitions=definitions) +@contextmanager +def _open_component_parent( + location: NeXusLocationSpec, + *, + nx_class: type[snx.NXobject], + definitions: Mapping | None | NoNewDefinitionsType = NoNewDefinitions, +) -> Generator[snx.Group, None, None]: + """Locate the parent group of a NeXus component.""" + file_path = location.filename + entry_name = location.entry_name + with _open_nexus_file(file_path, definitions=definitions) as f: + entry = _unique_child_group(f, snx.NXentry, entry_name) + if nx_class is snx.NXsample: + yield entry + else: + yield _unique_child_group(entry, snx.NXinstrument, None) + + def _unique_child_group( group: snx.Group, nx_class: type[snx.NXobject], name: str | None ) -> snx.Group: diff --git a/src/ess/reduce/nexus/types.py b/src/ess/reduce/nexus/types.py index 9df0635f..f8e00b61 100644 --- a/src/ess/reduce/nexus/types.py +++ b/src/ess/reduce/nexus/types.py @@ -112,6 +112,8 @@ class TransmissionRun(Generic[ScatteringRunType]): snx.NXdetector, snx.NXsample, snx.NXsource, + snx.NXdisk_chopper, + snx.NXcrystal, Monitor1, Monitor2, Monitor3, @@ -128,7 +130,7 @@ class NeXusName(sciline.Scope[Component, str], str): """Name of a component in a NeXus file.""" -class NeXusClass(sciline.Scope[Component, str], str): +class NeXusClass(sciline.Scope[Component, type], type): """NX_class of a component in a NeXus file.""" @@ -142,6 +144,12 @@ class NeXusComponent( """Raw data from a NeXus component.""" +class AllNeXusComponents( + sciline.ScopeTwoParams[Component, RunType, sc.DataGroup], sc.DataGroup +): + """Raw data from all NeXus components of one class.""" + + class NeXusData(sciline.ScopeTwoParams[Component, RunType, sc.DataArray], sc.DataArray): """ Data array loaded from an NXevent_data or NXdata group. @@ -227,9 +235,6 @@ class NeXusDataLocationSpec(NeXusLocationSpec, Generic[Component, RunType]): """NeXus filename and parameters to identify (parts of) detector data to load.""" -T = TypeVar('T', bound='NeXusTransformationChain') - - class NeXusTransformationChain( sciline.ScopeTwoParams[Component, RunType, snx.TransformationChain], snx.TransformationChain, @@ -257,3 +262,10 @@ def from_chain( raise ValueError(f"Expected scalar transformation, got {chain}") transform = chain.compute() return NeXusTransformation(value=transform) + + +class Choppers( + sciline.Scope[RunType, sc.DataGroup[sc.DataGroup[Any]]], + sc.DataGroup[sc.DataGroup[Any]], +): + """All choppers in a NeXus file.""" diff --git a/src/ess/reduce/nexus/workflow.py b/src/ess/reduce/nexus/workflow.py index 04994985..00d39f0c 100644 --- a/src/ess/reduce/nexus/workflow.py +++ b/src/ess/reduce/nexus/workflow.py @@ -13,12 +13,15 @@ import scippnexus as snx from scipp.constants import g from scipp.core import label_based_index_to_positional_index +from scippneutron.chopper import extract_chopper_from_nexus from . import _nexus_loader as nexus from .types import ( + AllNeXusComponents, CalibratedBeamline, CalibratedDetector, CalibratedMonitor, + Choppers, Component, DetectorBankSizes, DetectorData, @@ -96,6 +99,22 @@ def component_spec_by_name( ) +def disk_chopper_component_spec( + filename: NeXusFileSpec[RunType], +) -> NeXusComponentLocationSpec[snx.NXdisk_chopper, RunType]: + """Create a location spec for a disk chopper group in a NeXus file.""" + return NeXusComponentLocationSpec[snx.NXdisk_chopper, RunType]( + filename=filename.value + ) + + +def crystal_component_spec( + filename: NeXusFileSpec[RunType], +) -> NeXusComponentLocationSpec[snx.NXcrystal, RunType]: + """Create a location spec for a crystal / analyzer group in a NeXus file.""" + return NeXusComponentLocationSpec[snx.NXcrystal, RunType](filename=filename.value) + + def unique_component_spec( filename: NeXusFileSpec[RunType], ) -> NeXusComponentLocationSpec[UniqueComponent, RunType]: @@ -173,6 +192,14 @@ def nx_class_for_sample() -> NeXusClass[snx.NXsample]: return NeXusClass[snx.NXsample](snx.NXsample) +def nx_class_for_disk_chopper() -> NeXusClass[snx.NXdisk_chopper]: + return NeXusClass[snx.NXdisk_chopper](snx.NXdisk_chopper) + + +def nx_class_for_crystal() -> NeXusClass[snx.NXcrystal]: + return NeXusClass[snx.NXcrystal](snx.NXcrystal) + + def load_nexus_component( location: NeXusComponentLocationSpec[Component, RunType], nx_class: NeXusClass[Component], @@ -206,6 +233,27 @@ def load_nexus_component( ) +def load_all_nexus_components( + location: NeXusComponentLocationSpec[Component, RunType], + nx_class: NeXusClass[Component], +) -> AllNeXusComponents[Component, RunType]: + """ + Load all NeXus components of one class from one entry a file. + + This is equivalent to calling :func:`load_nexus_component` for every component. + + Parameters + ---------- + location: + Location spec for the source group. + nx_class: + NX_class to identify the components. + """ + return AllNeXusComponents[Component, RunType]( + nexus.load_all_components(location, nx_class=nx_class, definitions=definitions) + ) + + def load_nexus_data( location: NeXusDataLocationSpec[Component, RunType], ) -> NeXusData[Component, RunType]: @@ -465,6 +513,22 @@ def assemble_monitor_data( return MonitorData[RunType, MonitorType](_add_variances(da)) +def parse_disk_choppers( + choppers: AllNeXusComponents[snx.NXdisk_chopper, RunType], +) -> Choppers[RunType]: + """Convert the NeXus representation of a chopper to ours.""" + return Choppers[RunType]( + sc.DataGroup( + { + name: extract_chopper_from_nexus( + nexus.compute_component_position(chopper) + ) + for name, chopper in choppers.items() + } + ) + ) + + def _drop( children: dict[str, snx.Field | snx.Group], classes: tuple[snx.NXobject, ...] ) -> dict[str, snx.Field | snx.Group]: @@ -535,17 +599,22 @@ def _add_variances(da: sc.DataArray) -> sc.DataArray: file_path_to_file_spec, full_time_interval, component_spec_by_name, + disk_chopper_component_spec, + crystal_component_spec, unique_component_spec, # after component_spec_by_name, partially overrides get_transformation_chain, to_transformation, compute_position, load_nexus_data, load_nexus_component, + load_all_nexus_components, data_by_name, nx_class_for_detector, nx_class_for_monitor, nx_class_for_source, nx_class_for_sample, + nx_class_for_disk_chopper, + nx_class_for_crystal, ) _monitor_providers = ( @@ -562,6 +631,8 @@ def _add_variances(da: sc.DataArray) -> sc.DataArray: assemble_detector_data, ) +_chopper_providers = (parse_disk_choppers,) + def LoadMonitorWorkflow() -> sciline.Pipeline: """Generic workflow for loading monitor data from a NeXus file.""" @@ -605,7 +676,12 @@ def GenericNeXusWorkflow( if monitor_types is not None and run_types is None: raise ValueError("run_types must be specified if monitor_types is specified") wf = sciline.Pipeline( - (*_common_providers, *_monitor_providers, *_detector_providers) + ( + *_common_providers, + *_monitor_providers, + *_detector_providers, + *_chopper_providers, + ) ) wf[DetectorBankSizes] = DetectorBankSizes({}) wf[PreopenNeXusFile] = PreopenNeXusFile(False) diff --git a/tests/nexus/nexus_loader_test.py b/tests/nexus/nexus_loader_test.py index b8d7e9ef..67610aea 100644 --- a/tests/nexus/nexus_loader_test.py +++ b/tests/nexus/nexus_loader_test.py @@ -4,6 +4,7 @@ from contextlib import contextmanager from io import BytesIO from pathlib import Path +from typing import Any import numpy as np import pytest @@ -81,6 +82,29 @@ def _sample_data() -> sc.DataGroup: ) +def _choppers_data() -> sc.DataGroup[sc.DataGroup[Any]]: + return sc.DataGroup[sc.DataGroup]( + { + 'chopper_1': sc.DataGroup[Any]( + { + 'slit_edges': sc.array(dims=['dim_0'], values=[-5, 45], unit='deg'), + 'slit_height': sc.scalar(0.054, unit='m'), + 'slits': np.int64(1), # snx returns this type when loading + 'nexus_component_name': 'chopper_1', + } + ), + 'chopper_2': sc.DataGroup[Any]( + { + 'slit_edges': sc.array(dims=['dim_0'], values=[15, 60], unit='deg'), + 'slit_height': sc.scalar(0.07, unit='m'), + 'slits': np.int64(1), + 'nexus_component_name': 'chopper_2', + } + ), + } + ) + + def _write_transformation(group: snx.Group, offset: sc.Variable) -> None: group.create_field('depends_on', sc.scalar('transformations/t1')) transformations = group.create_class('transformations', snx.NXtransformations) @@ -131,6 +155,12 @@ def _write_nexus_data(store: Path | BytesIO) -> None: sample.create_field('chemical_formula', sample_data['chemical_formula']) sample.create_field('type', sample_data['type']) + for name, chopper in _choppers_data().items(): + chop = instrument.create_class(name, snx.NXdisk_chopper) + chop.create_field('slit_edges', chopper['slit_edges']) + chop.create_field('slit_height', chopper['slit_height']) + chop.create_field('slits', chopper['slits']) + @contextmanager def _file_store(request: pytest.FixtureRequest): @@ -210,6 +240,11 @@ def expected_sample() -> sc.DataGroup: return _sample_data() +@pytest.fixture +def expected_choppers() -> sc.DataGroup[sc.DataGroup[Any]]: + return _choppers_data() + + def test_load_data_loads_expected_event_data(nexus_file, expected_bank12): events = nexus.load_data( nexus_file, @@ -458,6 +493,16 @@ def test_load_sample(nexus_file, expected_sample, entry_name): sc.testing.assert_identical(sample, expected_sample) +@pytest.mark.parametrize('entry_name', [None, nexus.types.NeXusEntryName('entry-001')]) +def test_load_disk_choppers(nexus_file, expected_choppers, entry_name): + loc = NeXusLocationSpec( + filename=nexus_file, + entry_name=entry_name, + ) + choppers = nexus.load_all_components(loc, nx_class=snx.NXdisk_chopper) + sc.testing.assert_identical(choppers, expected_choppers) + + def test_extract_detector_data(): detector = sc.DataGroup( { diff --git a/tests/nexus/workflow_test.py b/tests/nexus/workflow_test.py index 4f0cac90..6349b631 100644 --- a/tests/nexus/workflow_test.py +++ b/tests/nexus/workflow_test.py @@ -9,6 +9,7 @@ from ess.reduce.nexus import compute_component_position, workflow from ess.reduce.nexus.types import ( BackgroundRun, + Choppers, DetectorData, Filename, Monitor1, @@ -541,6 +542,25 @@ def test_generic_nexus_workflow() -> None: assert da.dims == ('event_time_zero',) +def test_generic_nexus_workflow_load_choppers() -> None: + wf = GenericNeXusWorkflow() + wf[Filename[SampleRun]] = data.bifrost_simulated_elastic() + choppers = wf.compute(Choppers[SampleRun]) + + assert choppers.keys() == { + '005_PulseShapingChopper', + '006_PulseShapingChopper2', + '019_FOC1', + '048_FOC2', + '095_BWC1', + '096_BWC2', + } + chopper = choppers['005_PulseShapingChopper'] + assert 'position' in chopper + assert 'rotation_speed' in chopper + assert chopper['slit_edges'].shape == (2,) + + def test_generic_nexus_workflow_raises_if_monitor_types_but_not_run_types_given() -> ( None ): From 75a76dd587174264b62fc06f7caf3ba0aa5d6fc0 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Mon, 18 Nov 2024 16:08:27 +0100 Subject: [PATCH 3/5] Add ability to load all analyzers from NeXus --- src/ess/reduce/nexus/types.py | 7 ++++++ src/ess/reduce/nexus/workflow.py | 18 ++++++++++++++ tests/nexus/nexus_loader_test.py | 41 ++++++++++++++++++++++++++++++++ tests/nexus/workflow_test.py | 13 ++++++++++ 4 files changed, 79 insertions(+) diff --git a/src/ess/reduce/nexus/types.py b/src/ess/reduce/nexus/types.py index f8e00b61..ba52fdcb 100644 --- a/src/ess/reduce/nexus/types.py +++ b/src/ess/reduce/nexus/types.py @@ -269,3 +269,10 @@ class Choppers( sc.DataGroup[sc.DataGroup[Any]], ): """All choppers in a NeXus file.""" + + +class Analyzers( + sciline.Scope[RunType, sc.DataGroup[sc.DataGroup[Any]]], + sc.DataGroup[sc.DataGroup[Any]], +): + """All analyzers in a NeXus file.""" diff --git a/src/ess/reduce/nexus/workflow.py b/src/ess/reduce/nexus/workflow.py index 00d39f0c..0b81d77e 100644 --- a/src/ess/reduce/nexus/workflow.py +++ b/src/ess/reduce/nexus/workflow.py @@ -18,6 +18,7 @@ from . import _nexus_loader as nexus from .types import ( AllNeXusComponents, + Analyzers, CalibratedBeamline, CalibratedDetector, CalibratedMonitor, @@ -529,6 +530,20 @@ def parse_disk_choppers( ) +def parse_analyzers( + analyzers: AllNeXusComponents[snx.NXcrystal, RunType], +) -> Analyzers[RunType]: + """Convert the NeXus representation of an analyzer to ours.""" + return Analyzers[RunType]( + sc.DataGroup( + { + name: nexus.compute_component_position(analyzer) + for name, analyzer in analyzers.items() + } + ) + ) + + def _drop( children: dict[str, snx.Field | snx.Group], classes: tuple[snx.NXobject, ...] ) -> dict[str, snx.Field | snx.Group]: @@ -633,6 +648,8 @@ def _add_variances(da: sc.DataArray) -> sc.DataArray: _chopper_providers = (parse_disk_choppers,) +_analyzer_providers = (parse_analyzers,) + def LoadMonitorWorkflow() -> sciline.Pipeline: """Generic workflow for loading monitor data from a NeXus file.""" @@ -681,6 +698,7 @@ def GenericNeXusWorkflow( *_monitor_providers, *_detector_providers, *_chopper_providers, + *_analyzer_providers, ) ) wf[DetectorBankSizes] = DetectorBankSizes({}) diff --git a/tests/nexus/nexus_loader_test.py b/tests/nexus/nexus_loader_test.py index 67610aea..c01086e6 100644 --- a/tests/nexus/nexus_loader_test.py +++ b/tests/nexus/nexus_loader_test.py @@ -105,6 +105,27 @@ def _choppers_data() -> sc.DataGroup[sc.DataGroup[Any]]: ) +def _analyzers_data() -> sc.DataGroup[sc.DataGroup[Any]]: + return sc.DataGroup[sc.DataGroup[Any]]( + { + 'analyzer_B': sc.DataGroup[Any]( + { + 'd_spacing': sc.scalar(3.355, unit='angstrom'), + 'usage': 'Bragg', + 'nexus_component_name': 'analyzer_B', + } + ), + 'analyzer_A': sc.DataGroup[Any]( + { + 'd_spacing': sc.scalar(3.104, unit='angstrom'), + 'usage': 'Bragg', + 'nexus_component_name': 'analyzer_A', + } + ), + } + ) + + def _write_transformation(group: snx.Group, offset: sc.Variable) -> None: group.create_field('depends_on', sc.scalar('transformations/t1')) transformations = group.create_class('transformations', snx.NXtransformations) @@ -161,6 +182,11 @@ def _write_nexus_data(store: Path | BytesIO) -> None: chop.create_field('slit_height', chopper['slit_height']) chop.create_field('slits', chopper['slits']) + for name, analyzer in _analyzers_data().items(): + ana = instrument.create_class(name, snx.NXcrystal) + ana.create_field('d_spacing', analyzer['d_spacing']) + ana.create_field('usage', analyzer['usage']) + @contextmanager def _file_store(request: pytest.FixtureRequest): @@ -245,6 +271,11 @@ def expected_choppers() -> sc.DataGroup[sc.DataGroup[Any]]: return _choppers_data() +@pytest.fixture +def expected_analyzers() -> sc.DataGroup[sc.DataGroup[Any]]: + return _analyzers_data() + + def test_load_data_loads_expected_event_data(nexus_file, expected_bank12): events = nexus.load_data( nexus_file, @@ -503,6 +534,16 @@ def test_load_disk_choppers(nexus_file, expected_choppers, entry_name): sc.testing.assert_identical(choppers, expected_choppers) +@pytest.mark.parametrize('entry_name', [None, nexus.types.NeXusEntryName('entry-001')]) +def test_load_analyzers(nexus_file, expected_analyzers, entry_name): + loc = NeXusLocationSpec( + filename=nexus_file, + entry_name=entry_name, + ) + analyzers = nexus.load_all_components(loc, nx_class=snx.NXcrystal) + sc.testing.assert_identical(analyzers, expected_analyzers) + + def test_extract_detector_data(): detector = sc.DataGroup( { diff --git a/tests/nexus/workflow_test.py b/tests/nexus/workflow_test.py index 6349b631..5be1b663 100644 --- a/tests/nexus/workflow_test.py +++ b/tests/nexus/workflow_test.py @@ -8,6 +8,7 @@ from ess.reduce import data from ess.reduce.nexus import compute_component_position, workflow from ess.reduce.nexus.types import ( + Analyzers, BackgroundRun, Choppers, DetectorData, @@ -561,6 +562,18 @@ def test_generic_nexus_workflow_load_choppers() -> None: assert chopper['slit_edges'].shape == (2,) +def test_generic_nexus_workflow_load_analyzers() -> None: + wf = GenericNeXusWorkflow() + wf[Filename[SampleRun]] = data.bifrost_simulated_elastic() + analyzers = wf.compute(Analyzers[SampleRun]) + + assert len(analyzers) == 45 + analyzer = analyzers['144_channel_2_1_monochromator'] + assert 'position' in analyzer + assert analyzer['d_spacing'].ndim == 0 + assert analyzer['usage'] == 'Bragg' + + def test_generic_nexus_workflow_raises_if_monitor_types_but_not_run_types_given() -> ( None ): From 5622473c902413cf174ccdcc3a8b7deb223c8dec Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 20 Nov 2024 14:29:26 +0100 Subject: [PATCH 4/5] Use DataGroup.apply --- src/ess/reduce/nexus/workflow.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/src/ess/reduce/nexus/workflow.py b/src/ess/reduce/nexus/workflow.py index 0b81d77e..033906fe 100644 --- a/src/ess/reduce/nexus/workflow.py +++ b/src/ess/reduce/nexus/workflow.py @@ -519,13 +519,10 @@ def parse_disk_choppers( ) -> Choppers[RunType]: """Convert the NeXus representation of a chopper to ours.""" return Choppers[RunType]( - sc.DataGroup( - { - name: extract_chopper_from_nexus( - nexus.compute_component_position(chopper) - ) - for name, chopper in choppers.items() - } + choppers.apply( + lambda chopper: extract_chopper_from_nexus( + nexus.compute_component_position(chopper) + ) ) ) @@ -534,14 +531,7 @@ def parse_analyzers( analyzers: AllNeXusComponents[snx.NXcrystal, RunType], ) -> Analyzers[RunType]: """Convert the NeXus representation of an analyzer to ours.""" - return Analyzers[RunType]( - sc.DataGroup( - { - name: nexus.compute_component_position(analyzer) - for name, analyzer in analyzers.items() - } - ) - ) + return Analyzers[RunType](analyzers.apply(nexus.compute_component_position)) def _drop( From ef9e639c91f7ddea39050d28142c2f8cbf0c09db Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 20 Nov 2024 14:30:15 +0100 Subject: [PATCH 5/5] Use dedicated types for AllComponent specs --- src/ess/reduce/nexus/_nexus_loader.py | 16 +++++++++------- src/ess/reduce/nexus/types.py | 18 ++++++++++++++++++ src/ess/reduce/nexus/workflow.py | 23 +++++++---------------- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/src/ess/reduce/nexus/_nexus_loader.py b/src/ess/reduce/nexus/_nexus_loader.py index 47782016..6d655391 100644 --- a/src/ess/reduce/nexus/_nexus_loader.py +++ b/src/ess/reduce/nexus/_nexus_loader.py @@ -13,7 +13,14 @@ import scippnexus as snx from ..logging import get_logger -from .types import FilePath, NeXusEntryName, NeXusFile, NeXusGroup, NeXusLocationSpec +from .types import ( + FilePath, + NeXusAllLocationSpec, + NeXusEntryName, + NeXusFile, + NeXusGroup, + NeXusLocationSpec, +) class NoNewDefinitionsType: ... @@ -41,17 +48,12 @@ def load_component( def load_all_components( - location: NeXusLocationSpec, + location: NeXusAllLocationSpec, *, nx_class: type[snx.NXobject], definitions: Mapping | None | NoNewDefinitionsType = NoNewDefinitions, ) -> sc.DataGroup: """Load all components of a given class from NeXus.""" - if location.component_name is not None: - raise ValueError( - "`location.component_name` must be None when loading all " - "components of a certain class." - ) with _open_component_parent( location, nx_class=nx_class, definitions=definitions ) as parent: diff --git a/src/ess/reduce/nexus/types.py b/src/ess/reduce/nexus/types.py index ba52fdcb..4ec6ef15 100644 --- a/src/ess/reduce/nexus/types.py +++ b/src/ess/reduce/nexus/types.py @@ -230,6 +230,24 @@ class NeXusComponentLocationSpec(NeXusLocationSpec, Generic[Component, RunType]) """ +@dataclass +class NeXusAllLocationSpec: + """ + NeXus parameters to identify all components of a class to load. + """ + + filename: FilePath | NeXusFile | NeXusGroup + entry_name: NeXusEntryName | None = None + selection: snx.typing.ScippIndex | slice = () + + +@dataclass +class NeXusAllComponentLocationSpec(NeXusAllLocationSpec, Generic[Component, RunType]): + """ + NeXus parameters to identify all components of a class to load. + """ + + @dataclass class NeXusDataLocationSpec(NeXusLocationSpec, Generic[Component, RunType]): """NeXus filename and parameters to identify (parts of) detector data to load.""" diff --git a/src/ess/reduce/nexus/workflow.py b/src/ess/reduce/nexus/workflow.py index 033906fe..d8febeaa 100644 --- a/src/ess/reduce/nexus/workflow.py +++ b/src/ess/reduce/nexus/workflow.py @@ -32,6 +32,7 @@ MonitorData, MonitorPositionOffset, MonitorType, + NeXusAllComponentLocationSpec, NeXusClass, NeXusComponent, NeXusComponentLocationSpec, @@ -100,20 +101,11 @@ def component_spec_by_name( ) -def disk_chopper_component_spec( +def all_component_spec( filename: NeXusFileSpec[RunType], -) -> NeXusComponentLocationSpec[snx.NXdisk_chopper, RunType]: - """Create a location spec for a disk chopper group in a NeXus file.""" - return NeXusComponentLocationSpec[snx.NXdisk_chopper, RunType]( - filename=filename.value - ) - - -def crystal_component_spec( - filename: NeXusFileSpec[RunType], -) -> NeXusComponentLocationSpec[snx.NXcrystal, RunType]: - """Create a location spec for a crystal / analyzer group in a NeXus file.""" - return NeXusComponentLocationSpec[snx.NXcrystal, RunType](filename=filename.value) +) -> NeXusAllComponentLocationSpec[Component, RunType]: + """Create a location spec for all components of a class in a NeXus file.""" + return NeXusAllComponentLocationSpec[Component, RunType](filename=filename.value) def unique_component_spec( @@ -235,7 +227,7 @@ def load_nexus_component( def load_all_nexus_components( - location: NeXusComponentLocationSpec[Component, RunType], + location: NeXusAllComponentLocationSpec[Component, RunType], nx_class: NeXusClass[Component], ) -> AllNeXusComponents[Component, RunType]: """ @@ -604,9 +596,8 @@ def _add_variances(da: sc.DataArray) -> sc.DataArray: file_path_to_file_spec, full_time_interval, component_spec_by_name, - disk_chopper_component_spec, - crystal_component_spec, unique_component_spec, # after component_spec_by_name, partially overrides + all_component_spec, get_transformation_chain, to_transformation, compute_position,