From ab8df4a72075e8fc7e0c4d5f188ee9a905fcb50c Mon Sep 17 00:00:00 2001
From: Jan-Lukas Wynen <jan-lukas.wynen@ess.eu>
Date: Tue, 21 Nov 2023 15:42:46 +0100
Subject: [PATCH 01/16] Begin implementing Sciline providers

---
 src/ess/diffraction/__init__.py               |  8 +++
 .../diffraction/external/powgen/__init__.py   |  3 +
 src/ess/diffraction/external/powgen/data.py   | 54 ++++++++++++++++
 src/ess/diffraction/filtering.py              | 22 +++++++
 src/ess/diffraction/types.py                  | 63 +++++++++++++++++++
 5 files changed, 150 insertions(+)
 create mode 100644 src/ess/diffraction/types.py

diff --git a/src/ess/diffraction/__init__.py b/src/ess/diffraction/__init__.py
index 517b6ae..ab1963e 100644
--- a/src/ess/diffraction/__init__.py
+++ b/src/ess/diffraction/__init__.py
@@ -7,6 +7,7 @@
 
 import importlib.metadata
 
+from . import filtering
 from .corrections import normalize_by_monitor, normalize_by_vanadium
 from .grouping import group_by_two_theta
 from .smoothing import lowpass
@@ -18,6 +19,13 @@
 
 del importlib
 
+providers = (*filtering.providers,)
+"""Sciline providers for setting up a diffraction pipeline.
+
+These implement basic diffraction data-reduction functionality and need to be
+extended with instrument-specific and sub-technique-specific providers.
+"""
+
 __all__ = [
     'lowpass',
     'group_by_two_theta',
diff --git a/src/ess/diffraction/external/powgen/__init__.py b/src/ess/diffraction/external/powgen/__init__.py
index cc8291d..225c1b8 100644
--- a/src/ess/diffraction/external/powgen/__init__.py
+++ b/src/ess/diffraction/external/powgen/__init__.py
@@ -10,6 +10,9 @@
 from . import beamline, data
 from .instrument_view import instrument_view
 
+providers = (*data.providers,)
+"""Sciline Providers for POWGEN-specific functionality."""
+
 __all__ = [
     'beamline',
     'data',
diff --git a/src/ess/diffraction/external/powgen/data.py b/src/ess/diffraction/external/powgen/data.py
index 0fdb52b..b5b7042 100644
--- a/src/ess/diffraction/external/powgen/data.py
+++ b/src/ess/diffraction/external/powgen/data.py
@@ -1,5 +1,20 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright (c) 2023 Scipp contributors (https://github.com/scipp)
+
+"""Utilities for loading example data for POWGEN."""
+
+import scipp as sc
+
+from ...types import (
+    DetectorInfo,
+    Filename,
+    ProtonCharge,
+    RawData,
+    RawDataAndMetadata,
+    RunType,
+    SampleRun,
+)
+
 _version = '1'
 
 __all__ = ['get_path']
@@ -71,3 +86,42 @@ def vanadium_file() -> str:
 def calibration_file() -> str:
     (path,) = get_path('PG3_FERNS_d4832_2011_08_24.zip', unzip=True)
     return path
+
+
+def pooch_load(filename: Filename[RunType]) -> RawDataAndMetadata[RunType]:
+    """Load a file with pooch.
+
+    If the file is a zip archive, it is extracted and a path to the contained
+    file is returned.
+
+    The loaded data holds both the events and any metadata from the file.
+    """
+    if filename.endswith('.zip'):
+        (path,) = get_path(filename, unzip=True)
+    else:
+        path = get_path(filename)
+    return RawDataAndMetadata[RunType](sc.io.load_hdf5(path))
+
+
+def extract_raw_data(dg: RawDataAndMetadata[RunType]) -> RawData[RunType]:
+    """Return the events from a loaded data group."""
+    return RawData[RunType](dg['data'])
+
+
+def extract_detector_info(dg: RawDataAndMetadata[SampleRun]) -> DetectorInfo:
+    """Return the detector info from a loaded data group."""
+    return DetectorInfo(dg['detector_info'])
+
+
+def extract_proton_charge(dg: RawDataAndMetadata[RunType]) -> ProtonCharge[RunType]:
+    """Return the proton charge from a loaded data group."""
+    return ProtonCharge[RunType](dg['proton_charge'])
+
+
+providers = (
+    pooch_load,
+    extract_raw_data,
+    extract_detector_info,
+    extract_proton_charge,
+)
+"""Sciline Providers for loading POWGEN data."""
diff --git a/src/ess/diffraction/filtering.py b/src/ess/diffraction/filtering.py
index e548e4e..c5f8427 100644
--- a/src/ess/diffraction/filtering.py
+++ b/src/ess/diffraction/filtering.py
@@ -11,6 +11,8 @@
 
 import scipp as sc
 
+from .types import FilteredData, RawData, RunType, TofCroppedData, ValidTofRange
+
 
 def _equivalent_bin_indices(a, b) -> bool:
     a_begin = a.bins.constituents['begin'].flatten(to='')
@@ -67,3 +69,23 @@ def remove_bad_pulses(
     filtered = filtered.squeeze('good_pulse').copy(deep=False)
     del filtered.coords['good_pulse']
     return filtered
+
+
+def crop_tof(
+    data: RawData[RunType], tof_range: ValidTofRange
+) -> TofCroppedData[RunType]:
+    return TofCroppedData[RunType](
+        data.bin(tof=tof_range.to(unit=data.coords['tof'].unit))
+    )
+
+
+def filter_events(data: TofCroppedData[RunType]) -> FilteredData[RunType]:
+    # TODO this needs to filter by proton charge once we know how
+    return FilteredData[RunType](data)
+
+
+providers = (
+    crop_tof,
+    filter_events,
+)
+"""Sciline providers for event filtering."""
diff --git a/src/ess/diffraction/types.py b/src/ess/diffraction/types.py
new file mode 100644
index 0000000..5754367
--- /dev/null
+++ b/src/ess/diffraction/types.py
@@ -0,0 +1,63 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2023 Scipp contributors (https://github.com/scipp)
+
+"""This module defines the domain types uses in ess.diffraction.
+
+The domain types are used to define parameters and to request results from a Sciline
+pipeline.
+"""
+
+from typing import NewType, TypeVar
+
+import sciline
+import scipp as sc
+
+# 1 TypeVars used to parametrize the generic parts of the workflow
+
+# 1.1 Run types
+EmptyInstrumentRun = NewType('EmptyInstrumentRun', int)
+"""Empty instrument run."""
+SampleRun = NewType('SampleRun', int)
+"""Sample run."""
+VanadiumRun = NewType('VanadiumRun', int)
+"""Vanadium run."""
+RunType = TypeVar('RunType', EmptyInstrumentRun, SampleRun, VanadiumRun)
+"""TypeVar used for specifying the run."""
+
+
+# 2 Workflow parameters
+
+
+class Filename(sciline.Scope[RunType, str], str):
+    """Filename of a run."""
+
+
+ValidTofRange = NewType('ValidTofRange', sc.Variable)
+"""Min and max tof value of the instrument."""
+
+# 3 Workflow (intermediate) results
+
+# This is Mantid-specific and can probably be removed when the POWGEN
+# workflow is removed.
+DetectorInfo = NewType('DetectorInfo', sc.Dataset)
+"""Mapping between detector numbers and spectra."""
+
+
+class FilteredData(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
+    """Raw data without invalid events."""
+
+
+class ProtonCharge(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
+    """Time-dependent proton charge."""
+
+
+class RawData(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
+    """Raw data."""
+
+
+class RawDataAndMetadata(sciline.Scope[RunType, sc.DataGroup], sc.DataGroup):
+    """Raw data and associated metadata."""
+
+
+class TofCroppedData(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
+    """Raw data cropped to the valid TOF range."""

From 5dc6ba55b36100521011ccb0acb787ec8f224865 Mon Sep 17 00:00:00 2001
From: Jan-Lukas Wynen <jan-lukas.wynen@ess.eu>
Date: Tue, 21 Nov 2023 16:19:43 +0100
Subject: [PATCH 02/16] Pipeline up to conversion to dspacing

---
 src/ess/diffraction/__init__.py               |  6 ++--
 .../{corrections.py => correction.py}         | 30 +++++++++++++++++++
 .../diffraction/external/powgen/__init__.py   |  5 +++-
 .../diffraction/external/powgen/beamline.py   | 16 ++++++++--
 src/ess/diffraction/external/powgen/data.py   | 23 +++++++++++++-
 src/ess/diffraction/powder/__init__.py        |  9 ++++--
 .../powder/{conversions.py => conversion.py}  | 14 ++++++---
 .../powder/{corrections.py => correction.py}  |  0
 src/ess/diffraction/types.py                  | 23 ++++++++++++++
 9 files changed, 114 insertions(+), 12 deletions(-)
 rename src/ess/diffraction/{corrections.py => correction.py} (78%)
 rename src/ess/diffraction/powder/{conversions.py => conversion.py} (91%)
 rename src/ess/diffraction/powder/{corrections.py => correction.py} (100%)

diff --git a/src/ess/diffraction/__init__.py b/src/ess/diffraction/__init__.py
index ab1963e..eaa309b 100644
--- a/src/ess/diffraction/__init__.py
+++ b/src/ess/diffraction/__init__.py
@@ -8,7 +8,8 @@
 import importlib.metadata
 
 from . import filtering
-from .corrections import normalize_by_monitor, normalize_by_vanadium
+from .correction import normalize_by_monitor, normalize_by_vanadium
+from .correction import providers as correction_providers
 from .grouping import group_by_two_theta
 from .smoothing import lowpass
 
@@ -19,12 +20,13 @@
 
 del importlib
 
-providers = (*filtering.providers,)
+providers = (*filtering.providers, *correction_providers)
 """Sciline providers for setting up a diffraction pipeline.
 
 These implement basic diffraction data-reduction functionality and need to be
 extended with instrument-specific and sub-technique-specific providers.
 """
+del correction_providers
 
 __all__ = [
     'lowpass',
diff --git a/src/ess/diffraction/corrections.py b/src/ess/diffraction/correction.py
similarity index 78%
rename from src/ess/diffraction/corrections.py
rename to src/ess/diffraction/correction.py
index 2fe5157..aaffc5a 100644
--- a/src/ess/diffraction/corrections.py
+++ b/src/ess/diffraction/correction.py
@@ -7,6 +7,12 @@
 
 from .logging import get_logger
 from .smoothing import lowpass
+from .types import (
+    AccumulatedProtonCharge,
+    FilteredData,
+    NormalizedByProtonCharge,
+    RunType,
+)
 
 
 def normalize_by_monitor(
@@ -86,3 +92,27 @@ def normalize_by_vanadium(
     # with a large scale if the proton charges in data and vanadium were
     # measured with different units.
     return (data.bins / norm).to(unit='one', copy=False)
+
+
+def normalize_by_proton_charge(
+    data: FilteredData[RunType], proton_charge: AccumulatedProtonCharge[RunType]
+) -> NormalizedByProtonCharge[RunType]:
+    """Normalize data by an accumulated proton charge.
+
+    Parameters
+    ----------
+    data:
+        Un-normalized data array as events or a histogram.
+    proton_charge:
+        Accumulated proton charge over the entire run.
+
+    Returns
+    -------
+    :
+        ``data / proton_charge``
+    """
+    return NormalizedByProtonCharge[RunType](data / proton_charge)
+
+
+providers = (normalize_by_proton_charge,)
+"""Sciline providers for diffraction corrections."""
diff --git a/src/ess/diffraction/external/powgen/__init__.py b/src/ess/diffraction/external/powgen/__init__.py
index 225c1b8..f9d9711 100644
--- a/src/ess/diffraction/external/powgen/__init__.py
+++ b/src/ess/diffraction/external/powgen/__init__.py
@@ -10,7 +10,10 @@
 from . import beamline, data
 from .instrument_view import instrument_view
 
-providers = (*data.providers,)
+providers = (
+    *beamline.providers,
+    *data.providers,
+)
 """Sciline Providers for POWGEN-specific functionality."""
 
 __all__ = [
diff --git a/src/ess/diffraction/external/powgen/beamline.py b/src/ess/diffraction/external/powgen/beamline.py
index 958b589..e0b4248 100644
--- a/src/ess/diffraction/external/powgen/beamline.py
+++ b/src/ess/diffraction/external/powgen/beamline.py
@@ -6,10 +6,12 @@
 
 import scipp as sc
 
+from ...types import CalibrationData, DetectorInfo, RawCalibrationData
+
 
 def map_detector_to_spectrum(
-    data: sc.DataArray, *, detector_info: sc.DataArray
-) -> sc.DataArray:
+    data: sc.Dataset, *, detector_info: sc.Dataset
+) -> sc.Dataset:
     """
     Transform 'detector' coords to 'spectrum'.
 
@@ -44,3 +46,13 @@ def map_detector_to_spectrum(
     )
 
     return out.rename_dims({'detector': 'spectrum'})
+
+
+def preprocess_calibration_data(
+    data: RawCalibrationData, detector_info: DetectorInfo
+) -> CalibrationData:
+    return CalibrationData(map_detector_to_spectrum(data, detector_info=detector_info))
+
+
+providers = (preprocess_calibration_data,)
+"""Sciline providers for POWGEN beamline processing."""
diff --git a/src/ess/diffraction/external/powgen/data.py b/src/ess/diffraction/external/powgen/data.py
index b5b7042..cfdea8d 100644
--- a/src/ess/diffraction/external/powgen/data.py
+++ b/src/ess/diffraction/external/powgen/data.py
@@ -6,9 +6,12 @@
 import scipp as sc
 
 from ...types import (
+    AccumulatedProtonCharge,
+    CalibrationFilename,
     DetectorInfo,
     Filename,
     ProtonCharge,
+    RawCalibrationData,
     RawData,
     RawDataAndMetadata,
     RunType,
@@ -103,6 +106,15 @@ def pooch_load(filename: Filename[RunType]) -> RawDataAndMetadata[RunType]:
     return RawDataAndMetadata[RunType](sc.io.load_hdf5(path))
 
 
+def pooch_load_calibration(filename: CalibrationFilename) -> RawCalibrationData:
+    """Load the calibration data for the POWGEN test data."""
+    if filename.endswith('.zip'):
+        (path,) = get_path(filename, unzip=True)
+    else:
+        path = get_path(filename)
+    return RawCalibrationData(sc.io.load_hdf5(path))
+
+
 def extract_raw_data(dg: RawDataAndMetadata[RunType]) -> RawData[RunType]:
     """Return the events from a loaded data group."""
     return RawData[RunType](dg['data'])
@@ -118,10 +130,19 @@ def extract_proton_charge(dg: RawDataAndMetadata[RunType]) -> ProtonCharge[RunTy
     return ProtonCharge[RunType](dg['proton_charge'])
 
 
+def extract_accumulated_proton_charge(
+    data: RawData[SampleRun],
+) -> AccumulatedProtonCharge[SampleRun]:
+    """Return the stored accumulated proton charge from a loaded data group."""
+    return AccumulatedProtonCharge[SampleRun](data.coords['gd_prtn_chrg'])
+
+
 providers = (
     pooch_load,
-    extract_raw_data,
+    pooch_load_calibration,
+    extract_accumulated_proton_charge,
     extract_detector_info,
     extract_proton_charge,
+    extract_raw_data,
 )
 """Sciline Providers for loading POWGEN data."""
diff --git a/src/ess/diffraction/powder/__init__.py b/src/ess/diffraction/powder/__init__.py
index 7107a8d..b6f8268 100644
--- a/src/ess/diffraction/powder/__init__.py
+++ b/src/ess/diffraction/powder/__init__.py
@@ -3,7 +3,12 @@
 """
 Components for powder diffraction experiments.
 """
-from .conversions import to_dspacing_with_calibration
-from .corrections import merge_calibration
+from .conversion import providers as conversion_providers
+from .conversion import to_dspacing_with_calibration
+from .correction import merge_calibration
+
+providers = (*conversion_providers,)
+"""Sciline providers for powder diffraction."""
+del conversion_providers
 
 __all__ = ['merge_calibration', 'to_dspacing_with_calibration']
diff --git a/src/ess/diffraction/powder/conversions.py b/src/ess/diffraction/powder/conversion.py
similarity index 91%
rename from src/ess/diffraction/powder/conversions.py
rename to src/ess/diffraction/powder/conversion.py
index ff2cbf9..abc225a 100644
--- a/src/ess/diffraction/powder/conversions.py
+++ b/src/ess/diffraction/powder/conversion.py
@@ -10,7 +10,8 @@
 import scipp as sc
 
 from ..logging import get_logger
-from .corrections import merge_calibration
+from ..types import CalibrationData, DspacingData, NormalizedByProtonCharge, RunType
+from .correction import merge_calibration
 
 
 def _dspacing_from_diff_calibration_generic_impl(t, t0, a, c):
@@ -94,8 +95,9 @@ def _consume_positions(position, sample_position, source_position):
 
 
 def to_dspacing_with_calibration(
-    data: sc.DataArray, *, calibration: Optional[sc.Dataset] = None
-) -> sc.DataArray:
+    data: NormalizedByProtonCharge[RunType],
+    calibration: Optional[CalibrationData] = None,
+) -> DspacingData[RunType]:
     """
     Transform coordinates to d-spacing from calibration parameters.
 
@@ -148,4 +150,8 @@ def to_dspacing_with_calibration(
 
     out = out.transform_coords('dspacing', graph=graph, keep_intermediate=False)
     out.coords.pop('_tag_positions_consumed', None)
-    return out
+    return DspacingData[RunType](out)
+
+
+providers = (to_dspacing_with_calibration,)
+"""Sciline providers for coordinate transformations."""
diff --git a/src/ess/diffraction/powder/corrections.py b/src/ess/diffraction/powder/correction.py
similarity index 100%
rename from src/ess/diffraction/powder/corrections.py
rename to src/ess/diffraction/powder/correction.py
diff --git a/src/ess/diffraction/types.py b/src/ess/diffraction/types.py
index 5754367..e3336fa 100644
--- a/src/ess/diffraction/types.py
+++ b/src/ess/diffraction/types.py
@@ -27,6 +27,9 @@
 
 # 2 Workflow parameters
 
+CalibrationFilename = NewType('CalibrationFilename', str)
+"""Filename of the instrument calibration file."""
+
 
 class Filename(sciline.Scope[RunType, str], str):
     """Filename of a run."""
@@ -37,20 +40,40 @@ class Filename(sciline.Scope[RunType, str], str):
 
 # 3 Workflow (intermediate) results
 
+
+class AccumulatedProtonCharge(sciline.Scope[RunType, sc.Variable], sc.Variable):
+    """Total proton charge."""
+
+
+CalibrationData = NewType('CalibrationData', sc.Dataset)
+"""Detector calibration data."""
+
 # This is Mantid-specific and can probably be removed when the POWGEN
 # workflow is removed.
 DetectorInfo = NewType('DetectorInfo', sc.Dataset)
 """Mapping between detector numbers and spectra."""
 
 
+class DspacingData(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
+    """Data converted to d-spacing."""
+
+
 class FilteredData(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
     """Raw data without invalid events."""
 
 
+class NormalizedByProtonCharge(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
+    """Data that has been normalized by proton charge."""
+
+
 class ProtonCharge(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
     """Time-dependent proton charge."""
 
 
+RawCalibrationData = NewType('CalibrationData', sc.Dataset)
+"""Calibration data as loaded from file, needs preprocessing before using."""
+
+
 class RawData(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
     """Raw data."""
 

From d2c7bc4d3e89737b0222adbb5ae7fe9c524a9560 Mon Sep 17 00:00:00 2001
From: Jan-Lukas Wynen <jan-lukas.wynen@ess.eu>
Date: Wed, 22 Nov 2023 15:32:51 +0100
Subject: [PATCH 03/16] Full pipeline

---
 src/ess/diffraction/__init__.py             | 12 +++++++---
 src/ess/diffraction/correction.py           | 13 ++++++++---
 src/ess/diffraction/external/powgen/data.py | 23 ++++++++++++++-----
 src/ess/diffraction/grouping.py             | 25 +++++++++++++++++++++
 src/ess/diffraction/types.py                | 25 ++++++++++++++++++++-
 src/ess/diffraction/uncertainty.py          | 18 +++++++++++++++
 6 files changed, 103 insertions(+), 13 deletions(-)
 create mode 100644 src/ess/diffraction/uncertainty.py

diff --git a/src/ess/diffraction/__init__.py b/src/ess/diffraction/__init__.py
index eaa309b..d246cfb 100644
--- a/src/ess/diffraction/__init__.py
+++ b/src/ess/diffraction/__init__.py
@@ -7,10 +7,11 @@
 
 import importlib.metadata
 
-from . import filtering
+from . import filtering, uncertainty
 from .correction import normalize_by_monitor, normalize_by_vanadium
 from .correction import providers as correction_providers
 from .grouping import group_by_two_theta
+from .grouping import providers as grouping_providers
 from .smoothing import lowpass
 
 try:
@@ -20,13 +21,18 @@
 
 del importlib
 
-providers = (*filtering.providers, *correction_providers)
+providers = (
+    *filtering.providers,
+    *correction_providers,
+    *grouping_providers,
+    *uncertainty.providers,
+)
 """Sciline providers for setting up a diffraction pipeline.
 
 These implement basic diffraction data-reduction functionality and need to be
 extended with instrument-specific and sub-technique-specific providers.
 """
-del correction_providers
+del correction_providers, grouping_providers
 
 __all__ = [
     'lowpass',
diff --git a/src/ess/diffraction/correction.py b/src/ess/diffraction/correction.py
index aaffc5a..b094a8c 100644
--- a/src/ess/diffraction/correction.py
+++ b/src/ess/diffraction/correction.py
@@ -9,9 +9,14 @@
 from .smoothing import lowpass
 from .types import (
     AccumulatedProtonCharge,
+    DspacingBins,
     FilteredData,
+    MergedPixels,
     NormalizedByProtonCharge,
+    NormalizedByVanadium,
     RunType,
+    SampleRun,
+    VanadiumRun,
 )
 
 
@@ -68,8 +73,10 @@ def normalize_by_monitor(
 
 
 def normalize_by_vanadium(
-    data: sc.DataArray, *, vanadium: sc.DataArray, edges: sc.Variable
-) -> sc.DataArray:
+    data: MergedPixels[SampleRun],
+    vanadium: MergedPixels[VanadiumRun],
+    edges: DspacingBins,
+) -> NormalizedByVanadium:
     """
     Normalize sample data by a vanadium measurement.
 
@@ -114,5 +121,5 @@ def normalize_by_proton_charge(
     return NormalizedByProtonCharge[RunType](data / proton_charge)
 
 
-providers = (normalize_by_proton_charge,)
+providers = (normalize_by_proton_charge, normalize_by_vanadium)
 """Sciline providers for diffraction corrections."""
diff --git a/src/ess/diffraction/external/powgen/data.py b/src/ess/diffraction/external/powgen/data.py
index cfdea8d..8dc3179 100644
--- a/src/ess/diffraction/external/powgen/data.py
+++ b/src/ess/diffraction/external/powgen/data.py
@@ -14,8 +14,10 @@
     RawCalibrationData,
     RawData,
     RawDataAndMetadata,
+    RawDataWithvariances,
     RunType,
     SampleRun,
+    VanadiumRun,
 )
 
 _version = '1'
@@ -115,9 +117,17 @@ def pooch_load_calibration(filename: CalibrationFilename) -> RawCalibrationData:
     return RawCalibrationData(sc.io.load_hdf5(path))
 
 
-def extract_raw_data(dg: RawDataAndMetadata[RunType]) -> RawData[RunType]:
+# This can be generalized with https://github.com/scipp/sciline/issues/69.
+def extract_raw_data_sample(dg: RawDataAndMetadata[SampleRun]) -> RawData[SampleRun]:
     """Return the events from a loaded data group."""
-    return RawData[RunType](dg['data'])
+    return RawData[SampleRun](dg['data'])
+
+
+def extract_raw_data_vanadium(
+    dg: RawDataAndMetadata[VanadiumRun],
+) -> RawDataWithvariances[VanadiumRun]:
+    """Return the events from a loaded data group."""
+    return RawDataWithvariances[VanadiumRun](dg['data'])
 
 
 def extract_detector_info(dg: RawDataAndMetadata[SampleRun]) -> DetectorInfo:
@@ -131,10 +141,10 @@ def extract_proton_charge(dg: RawDataAndMetadata[RunType]) -> ProtonCharge[RunTy
 
 
 def extract_accumulated_proton_charge(
-    data: RawData[SampleRun],
-) -> AccumulatedProtonCharge[SampleRun]:
+    data: RawData[RunType],
+) -> AccumulatedProtonCharge[RunType]:
     """Return the stored accumulated proton charge from a loaded data group."""
-    return AccumulatedProtonCharge[SampleRun](data.coords['gd_prtn_chrg'])
+    return AccumulatedProtonCharge[RunType](data.coords['gd_prtn_chrg'])
 
 
 providers = (
@@ -143,6 +153,7 @@ def extract_accumulated_proton_charge(
     extract_accumulated_proton_charge,
     extract_detector_info,
     extract_proton_charge,
-    extract_raw_data,
+    extract_raw_data_sample,
+    extract_raw_data_vanadium,
 )
 """Sciline Providers for loading POWGEN data."""
diff --git a/src/ess/diffraction/grouping.py b/src/ess/diffraction/grouping.py
index 5f18b7a..fefd650 100644
--- a/src/ess/diffraction/grouping.py
+++ b/src/ess/diffraction/grouping.py
@@ -3,6 +3,15 @@
 import scipp as sc
 from scippneutron.conversion.graph import beamline
 
+from .types import (
+    DspacingBins,
+    DspacingData,
+    DspacingHistogram,
+    MergedPixels,
+    NormalizedByVanadium,
+    RunType,
+)
+
 
 def group_by_two_theta(data: sc.DataArray, *, edges: sc.Variable) -> sc.DataArray:
     """
@@ -23,3 +32,19 @@ def group_by_two_theta(data: sc.DataArray, *, edges: sc.Variable) -> sc.DataArra
     """
     out = data.transform_coords('two_theta', graph=beamline.beamline(scatter=True))
     return out.bin(two_theta=edges.to(unit=out.coords['two_theta'].unit, copy=False))
+
+
+def merge_all_pixels(data: DspacingData[RunType]) -> MergedPixels[RunType]:
+    """Combine all pixels (spectra) of the detector."""
+    return MergedPixels(data.bins.concat('spectrum'))
+
+
+def finalize_histogram(
+    data: NormalizedByVanadium, edges: DspacingBins
+) -> DspacingHistogram:
+    """Finalize the d-spacing histogram."""
+    return DspacingHistogram(data.hist(dspacing=edges))
+
+
+providers = (merge_all_pixels, finalize_histogram)
+"""Sciline providers for grouping pixels."""
diff --git a/src/ess/diffraction/types.py b/src/ess/diffraction/types.py
index e3336fa..666ee7e 100644
--- a/src/ess/diffraction/types.py
+++ b/src/ess/diffraction/types.py
@@ -30,6 +30,9 @@
 CalibrationFilename = NewType('CalibrationFilename', str)
 """Filename of the instrument calibration file."""
 
+DspacingBins = NewType('DSpacingBins', sc.Variable)
+"""Bin edges for d-spacing."""
+
 
 class Filename(sciline.Scope[RunType, str], str):
     """Filename of a run."""
@@ -58,19 +61,35 @@ class DspacingData(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
     """Data converted to d-spacing."""
 
 
+class DspacingDataWithoutVariances(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
+    """Data converted to d-spacing where variances where removed."""
+
+
+DspacingHistogram = NewType('DspacingHistogram', sc.DataArray)
+"""Histogrammed intensity vs d-spacing."""
+
+
 class FilteredData(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
     """Raw data without invalid events."""
 
 
+class MergedPixels(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
+    """Intensity vs d-spacing for all detector pixels combined."""
+
+
 class NormalizedByProtonCharge(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
     """Data that has been normalized by proton charge."""
 
 
+NormalizedByVanadium = NewType('NormalizedByVanadium', sc.DataArray)
+"""Data that has been normalized by a vanadium run."""
+
+
 class ProtonCharge(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
     """Time-dependent proton charge."""
 
 
-RawCalibrationData = NewType('CalibrationData', sc.Dataset)
+RawCalibrationData = NewType('RawCalibrationData', sc.Dataset)
 """Calibration data as loaded from file, needs preprocessing before using."""
 
 
@@ -78,6 +97,10 @@ class RawData(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
     """Raw data."""
 
 
+class RawDataWithvariances(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
+    """Raw data that has variances which need special handling."""
+
+
 class RawDataAndMetadata(sciline.Scope[RunType, sc.DataGroup], sc.DataGroup):
     """Raw data and associated metadata."""
 
diff --git a/src/ess/diffraction/uncertainty.py b/src/ess/diffraction/uncertainty.py
new file mode 100644
index 0000000..600663f
--- /dev/null
+++ b/src/ess/diffraction/uncertainty.py
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2023 Scipp contributors (https://github.com/scipp)
+"""Tools for handling statistical uncertainties."""
+
+from .types import RawData, RawDataWithvariances, VanadiumRun
+
+
+def drop_variances(data: RawDataWithvariances[VanadiumRun]) -> RawData[VanadiumRun]:
+    res = data.copy(deep=False)
+    if res.bins is not None:
+        res.bins.constituents['data'].variances = None
+    else:
+        res.variances = None
+    return RawData[VanadiumRun](res)
+
+
+providers = (drop_variances,)
+"""Sciline providers for handling statistical uncertainties."""

From 555557a100f09d5434b25c654534f3ebae8dd5aa Mon Sep 17 00:00:00 2001
From: Jan-Lukas Wynen <jan-lukas.wynen@ess.eu>
Date: Wed, 22 Nov 2023 15:50:47 +0100
Subject: [PATCH 04/16] Add grouping in two_theta

---
 src/ess/diffraction/grouping.py | 11 ++++++++---
 src/ess/diffraction/types.py    |  3 +++
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/ess/diffraction/grouping.py b/src/ess/diffraction/grouping.py
index fefd650..db5e1ac 100644
--- a/src/ess/diffraction/grouping.py
+++ b/src/ess/diffraction/grouping.py
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright (c) 2023 Scipp contributors (https://github.com/scipp)
-import scipp as sc
+
 from scippneutron.conversion.graph import beamline
 
 from .types import (
@@ -10,10 +10,13 @@
     MergedPixels,
     NormalizedByVanadium,
     RunType,
+    TwoThetaBins,
 )
 
 
-def group_by_two_theta(data: sc.DataArray, *, edges: sc.Variable) -> sc.DataArray:
+def group_by_two_theta(
+    data: DspacingData[RunType], edges: TwoThetaBins
+) -> MergedPixels[RunType]:
     """
     Group data into two_theta bins.
 
@@ -31,7 +34,9 @@ def group_by_two_theta(data: sc.DataArray, *, edges: sc.Variable) -> sc.DataArra
         `data` grouped into two_theta bins.
     """
     out = data.transform_coords('two_theta', graph=beamline.beamline(scatter=True))
-    return out.bin(two_theta=edges.to(unit=out.coords['two_theta'].unit, copy=False))
+    return MergedPixels[RunType](
+        out.bin(two_theta=edges.to(unit=out.coords['two_theta'].unit, copy=False))
+    )
 
 
 def merge_all_pixels(data: DspacingData[RunType]) -> MergedPixels[RunType]:
diff --git a/src/ess/diffraction/types.py b/src/ess/diffraction/types.py
index 666ee7e..3f36ce7 100644
--- a/src/ess/diffraction/types.py
+++ b/src/ess/diffraction/types.py
@@ -38,6 +38,9 @@ class Filename(sciline.Scope[RunType, str], str):
     """Filename of a run."""
 
 
+TwoThetaBins = NewType('TwoThetaBins', sc.Variable)
+"""Bin edges for 2theta."""
+
 ValidTofRange = NewType('ValidTofRange', sc.Variable)
 """Min and max tof value of the instrument."""
 

From 4c8e7a66a2ec99277ada770beb4e6b0fe1db4bda Mon Sep 17 00:00:00 2001
From: Jan-Lukas Wynen <jan-lukas.wynen@ess.eu>
Date: Fri, 24 Nov 2023 16:37:31 +0100
Subject: [PATCH 05/16] Convert docs to use sciline

---
 docs/api-reference/index.md               |   2 +
 docs/examples/POWGEN_data_reduction.ipynb | 365 +++++++++++
 docs/examples/index.md                    |   2 +-
 docs/examples/powgen_reduction.ipynb      | 697 ----------------------
 4 files changed, 368 insertions(+), 698 deletions(-)
 create mode 100644 docs/examples/POWGEN_data_reduction.ipynb
 delete mode 100644 docs/examples/powgen_reduction.ipynb

diff --git a/docs/api-reference/index.md b/docs/api-reference/index.md
index 06485f6..a2bf329 100644
--- a/docs/api-reference/index.md
+++ b/docs/api-reference/index.md
@@ -26,4 +26,6 @@
    :toctree: ../generated/modules
    :template: module-template.rst
    :recursive:
+
+   types
 ```
\ No newline at end of file
diff --git a/docs/examples/POWGEN_data_reduction.ipynb b/docs/examples/POWGEN_data_reduction.ipynb
new file mode 100644
index 0000000..8520004
--- /dev/null
+++ b/docs/examples/POWGEN_data_reduction.ipynb
@@ -0,0 +1,365 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "936a13e1-c9ea-4351-b29d-7dae9ad1e073",
+   "metadata": {},
+   "source": [
+    "# POWGEN data reduction\n",
+    "\n",
+    "## Introduction\n",
+    "\n",
+    "This notebook gives a concise overview of how to use the ESSDiffraction package with Sciline.\n",
+    "It uses a simple reduction workflow for the SNS [POWGEN](https://sns.gov/powgen) experiment.\n",
+    "\n",
+    "We begin with relevant imports:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b160f394-de0b-4b73-9aa5-c13ac3a1d707",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import scipp as sc\n",
+    "import scippneutron as scn\n",
+    "import sciline\n",
+    "\n",
+    "import ess.diffraction\n",
+    "from ess.diffraction import powder\n",
+    "from ess.diffraction.external import powgen\n",
+    "from ess.diffraction.types import *"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6270f661-099f-4f8a-9380-76050f73f251",
+   "metadata": {},
+   "source": [
+    "## Define reduction parameters\n",
+    "\n",
+    "We define a dictionary containing the reduction parameters.\n",
+    "The keys are types defined in [essdiffraction.types](../generated/modules/ess.diffraction.types.rst)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "67e8d584-016d-422e-add9-a61e5b1fe8bd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "params = {\n",
+    "    # Input data\n",
+    "    Filename[SampleRun]: 'PG3_4844_event.zip',\n",
+    "    Filename[VanadiumRun]: 'PG3_4866_event.zip',\n",
+    "    CalibrationFilename: 'PG3_FERNS_d4832_2011_08_24.zip',\n",
+    "\n",
+    "    # Crop data to this range in time-of-flight\n",
+    "    ValidTofRange: sc.array(dims=['tof'], values=[0.0, 16666.67], unit='us'),\n",
+    "\n",
+    "    # Edges for binning in d-spacing\n",
+    "    DspacingBins: sc.linspace('dspacing', 0.0, 2.3434, 200, unit='angstrom'),\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e3d9b9bd-6923-4124-954f-e92672845be2",
+   "metadata": {},
+   "source": [
+    "## Create pipeline using Sciline\n",
+    "\n",
+    "We use the basic providers available in `essdiffraction` as well as the specialised `powder` and `powgen` providers."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "35412429-c039-4591-b8b8-49b11a128b7f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "providers = [\n",
+    "    *ess.diffraction.providers,\n",
+    "    *powder.providers,\n",
+    "    *powgen.providers,\n",
+    "]\n",
+    "pipeline = sciline.Pipeline(\n",
+    "    providers,\n",
+    "    params=params,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b4af9055-385d-41a4-9f1c-35fd1e232565",
+   "metadata": {},
+   "source": [
+    "## Use the pipeline\n",
+    "\n",
+    "### Compute final result\n",
+    "\n",
+    "We can get the graph for computing the final d-spacing histogram:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d09bc8d5-4251-483f-bb45-aa8cf67f69b2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dspacing_histogram = pipeline.get(DspacingHistogram)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "67288e4a-5140-430d-a15a-7caa0f365904",
+   "metadata": {},
+   "source": [
+    "Before computing the result, we visualize the pipeline:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "11fea84f-aef5-4d31-94c4-1f710c7f8836",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# left-right layout works better for this graph\n",
+    "dspacing_histogram.visualize(graph_attr={'rankdir': 'LR'})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a3177aeb-51ae-4166-907b-1caa9cf751aa",
+   "metadata": {},
+   "source": [
+    "Now we compute the result:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "14663b68-c9b8-444a-bf06-7454b67f8d82",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "result = dspacing_histogram.compute()\n",
+    "result"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "94792ed7-cfba-4467-9b5e-91211c5f2d89",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "result.plot()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6335d026-d526-4ad4-bcda-7270e955d2c1",
+   "metadata": {},
+   "source": [
+    "### Save reduced data to file\n",
+    "\n",
+    "TODO"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0420ed62-aa60-466f-a823-f608ddc3abc5",
+   "metadata": {},
+   "source": [
+    "### Compute intermediate results\n",
+    "\n",
+    "For inspection and debugging purposes, we can also compute intermediate results.\n",
+    "To avoid repeated computation (including costly loading of files), we can request multiple results at once, including the final result, if desired.\n",
+    "For example:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "300d11ce-203b-4c65-a7f6-9df5132f9830",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "results = pipeline.compute((\n",
+    "    RawData[SampleRun],\n",
+    "    FilteredData[SampleRun],\n",
+    "    FilteredData[VanadiumRun],\n",
+    "))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b57d17b6-926a-4b75-8221-5ed27f59997d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "results[RawData[SampleRun]]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "57d72dd9-7f21-4691-b478-109194cc9c13",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "scn.instrument_view(results[RawData[SampleRun]].hist())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e5547f3e-908c-4dcb-a700-c4aa5098073c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tof_data = sc.DataGroup(\n",
+    "    sample=results[RawData[SampleRun]].bins.concat('spectrum'),\n",
+    "    vanadium=results[FilteredData[VanadiumRun]].bins.concat('spectrum'),\n",
+    ")\n",
+    "tof_data.hist(tof=100).plot()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4048b5ad-acfa-4584-8cf6-8ebe2cba6801",
+   "metadata": {},
+   "source": [
+    "## Group by scattering angle\n",
+    "\n",
+    "The above pipeline merges all instrument pixels to produce a 1d d-spacing curve.\n",
+    "If instead we want to group into $2\\theta$ bins, we can alter the pipeline by replacing the grouping step:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d24c8bf8-ad59-4211-ae6a-3ee29b0556a3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from ess.diffraction.grouping import group_by_two_theta, merge_all_pixels\n",
+    "\n",
+    "grouping_providers = list(providers)\n",
+    "grouping_providers.remove(merge_all_pixels)\n",
+    "grouping_providers = (*grouping_providers, group_by_two_theta)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3e83ed6c-46fc-4396-8331-3654993def94",
+   "metadata": {},
+   "source": [
+    "We also need to specify the grouping with a new parameter:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a4b68853-a70b-42d6-a8cc-58c77e83eaec",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "params[TwoThetaBins] = sc.linspace(dim='two_theta', unit='deg', start=25.0, stop=90.0, num=16)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ea65b6bd-b9fa-496d-bfa2-f459e33f75fc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "grouping_pipeline = sciline.Pipeline(\n",
+    "    grouping_providers,\n",
+    "    params=params\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "59ae5b45-8e11-4de1-bef5-c4b58df75804",
+   "metadata": {},
+   "source": [
+    "Inspect the graph to check that the new provider has been inserted:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a63e037d-3cfc-4ac9-963c-1e2d0881d482",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "grouped_dspacing = grouping_pipeline.get(DspacingHistogram)\n",
+    "grouped_dspacing.visualize(graph_attr={'rankdir': 'LR'})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2a8a0853-2829-49ab-b343-31deca3de31c",
+   "metadata": {},
+   "source": [
+    "Compute and plot the result:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bc73276a-dadb-4c21-9954-e69bebc2ff90",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "grouped_result = grouped_dspacing.compute()\n",
+    "grouped_result"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7ca04947-b019-4814-9467-4d0519d8d384",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "angle = sc.midpoints(grouped_result.coords['two_theta'])\n",
+    "sc.plot({\n",
+    "    f'{angle[group].value:.3f} {angle[group].unit}': grouped_result['two_theta', group]\n",
+    "    for group in range(2, 6)\n",
+    "})"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/docs/examples/index.md b/docs/examples/index.md
index 3463898..b1c64c0 100644
--- a/docs/examples/index.md
+++ b/docs/examples/index.md
@@ -5,5 +5,5 @@
 maxdepth: 2
 ---
 
-powgen_reduction
+POWGEN_data_reduction
 ```
diff --git a/docs/examples/powgen_reduction.ipynb b/docs/examples/powgen_reduction.ipynb
deleted file mode 100644
index da166b3..0000000
--- a/docs/examples/powgen_reduction.ipynb
+++ /dev/null
@@ -1,697 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "bedd5844-1c5c-4fcf-a311-e0625b5e23b7",
-   "metadata": {},
-   "source": [
-    "# Data reduction for POWGEN"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "094942ff-1bde-46c2-ae50-d00177eca3ad",
-   "metadata": {},
-   "source": [
-    "This notebook shows a basic reduction workflow for powder diffraction for the SNS [POWGEN](https://sns.gov/powgen) instrument.\n",
-    "It serves mainly to develop and present routines for powder diffraction and will eventually be removed in favor of a workflow for DREAM at ESS.\n",
-    "\n",
-    "**Note** that we load functions from `external` modules.\n",
-    "These modules will be removed when their ESS counterparts exist."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "b58104c6-a196-4576-b3f0-9fb6fb1216e9",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "import scipp as sc\n",
-    "import scippneutron as scn\n",
-    "import plopp as pp\n",
-    "\n",
-    "import ess\n",
-    "from ess.diffraction import powder\n",
-    "from ess import diffraction\n",
-    "from ess.diffraction.external import powgen"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "da4040e8-681e-4eb7-a205-af6a88539ffa",
-   "metadata": {},
-   "source": [
-    "## Load data"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "8db8e21f-f19c-4498-a4d1-d34142b2ba02",
-   "metadata": {},
-   "source": [
-    "Load the sample data.\n",
-    "\n",
-    "**Note:** We get the file name from `powgen.data`.\n",
-    "This module provides access to managed example files.\n",
-    "In the real world, we would need to find the file name in a different way.\n",
-    "But here, the data has been converted to a [Scipp HDF5 file](https://scipp.github.io/user-guide/reading-and-writing-files.html#HDF5)."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c1d52d95-727c-4d67-b1f9-dc8c4e345d15",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "sample_full = sc.io.load_hdf5(powgen.data.sample_file())"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "723704af-1dc1-4cce-a396-77a754b19d77",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "sample_full"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "bd6c57e1-183e-4f1d-ab3e-1c0cb0fd7e59",
-   "metadata": {},
-   "source": [
-    "The loaded data group contains some auxiliary detector info that we need later.\n",
-    "The events are"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "7216e1d6-844c-4622-9d84-91bb300b87a0",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "sample = sample_full['data']\n",
-    "sample"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "bab068d8-d02c-4bec-a9a3-2c3b713ab750",
-   "metadata": {},
-   "source": [
-    "## Inspect the raw data"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "330aaf4b-465f-47e9-bede-45183971f9f6",
-   "metadata": {},
-   "source": [
-    "We can plot the data array to get an idea of its contents."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "3848a30b-b872-465c-a1ca-2fad3a5110cb",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "sample.hist(spectrum=500, tof=400).plot()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "59585c24-349f-4c5b-ab35-b17772712688",
-   "metadata": {},
-   "source": [
-    "We can see how that data maps onto the detector by using POWGEN's instrument view."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "6e5a4d3c-cb93-4875-b2be-9e73df22771d",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "scn.instrument_view(sample.hist())"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "07912224-f459-4c42-95e0-58d99291a713",
-   "metadata": {},
-   "source": [
-    "## Filter out invalid events"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "a5121997-9522-4cd7-bbc4-468b38001aee",
-   "metadata": {},
-   "source": [
-    "The file contains events that cannot be associated with a specific pulse.\n",
-    "We can get a range of valid time-of-flight values from the instrument characterization file associated with the current run.\n",
-    "There is currently no mechanism in `scippneutron` or `ess` to load such a file as it is not clear if ESS will use this approach.\n",
-    "The values used below are taken from `PG3_characterization_2011_08_31-HR.txt` which is part of the sample files of Mantid.\n",
-    "See, e.g., [PowderDiffractionReduction](https://www.mantidproject.org/PowderDiffractionReduction).\n",
-    "\n",
-    "We remove all events that have a time-of-flight value outside the valid range:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ced73191-082a-4c7c-aa58-6111cd3cf1fd",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "sample = sample.bin(tof=sc.array(dims=['tof'], values=[0.0, 16666.67], unit='us'))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "2de9ce54-f972-474a-ae82-c9754b80719f",
-   "metadata": {},
-   "source": [
-    "## Normalize by proton charge"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "ebfdcccc-7638-4dbb-94f4-cf070d9ed28d",
-   "metadata": {},
-   "source": [
-    "Next, we normalize the data by the proton charge."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "85bd7848-a7db-42c6-974a-1eb5bf3e9860",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "sample /= sample.coords['gd_prtn_chrg']"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "b0e9bd7b-897f-41a1-80f7-1a12c8424c82",
-   "metadata": {},
-   "source": [
-    "We can check the unit of the event weight to see that the data was indeed divided by a charge."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "063b72ab-7b38-47fa-a180-e4cb659efb1f",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "sample.data.values[0].unit"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "52e9d8d7-474c-423b-baf4-e9e135f6e46d",
-   "metadata": {},
-   "source": [
-    "## Compute d-spacing"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "d9291753-2b26-4d5d-8396-f4d3a2a9abfc",
-   "metadata": {},
-   "source": [
-    "Here, we compute d-spacing using calibration parameters provided in an example file.\n",
-    "First, we load the calibration parameters.\n",
-    "\n",
-    "**Note:** ESS instruments will use a different, yet to be determined way of encoding calibration parameters."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "1ca5395a-4a43-4eaa-85fa-f43e6ac1a576",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "cal = sc.io.load_hdf5(powgen.data.calibration_file())"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "02c9608f-7b1d-412b-a4f0-df064b340916",
-   "metadata": {},
-   "source": [
-    "The calibration is loaded with a 'detector' dimension.\n",
-    "Compute the corresponding spectrum indices using the detector info loaded as part of the sample data."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "d9b964ee-43b3-4bc1-b079-dfede08d3208",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "cal = powgen.beamline.map_detector_to_spectrum(\n",
-    "    cal, detector_info=sample_full['detector_info']\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "59566cfb-e175-44ea-bd09-bd2324514405",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "cal"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "63553acb-4f78-4518-b5f8-6dc71dc616a5",
-   "metadata": {},
-   "source": [
-    "Now when can compute d-spacing for the sample using the calibration parameters."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "69388346-24ab-4035-b18d-b10bf274e4c4",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "sample_dspacing = powder.to_dspacing_with_calibration(sample, calibration=cal)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "bc226f10-71fe-4571-8391-db35028c4c76",
-   "metadata": {
-    "tags": []
-   },
-   "source": [
-    "## Vanadium correction"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "f670ce66-38cf-44ff-ad7c-d5dd30e3b09e",
-   "metadata": {},
-   "source": [
-    "Before we can process the d-spacing distribution further, we need to normalize the data by a vanadium measurement."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "b9494f15-1fb2-4236-95a7-800a04896e2f",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "vana_full = sc.io.load_hdf5(powgen.data.vanadium_file())"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "19cd62a0-94ef-4dd4-b00e-7d3d9d6c9d50",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "vana_full"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "cb9a0742-9887-4166-a8fe-29b3064c8e20",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "vana = vana_full['data']\n",
-    "vana"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "6280fb1c-e3a2-4042-b76d-e08e31187af7",
-   "metadata": {},
-   "source": [
-    "Now we process the vanadium data in a similar was as the sample data."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "2665c1ec-ae0b-440b-b4ab-60cb03f983a2",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "vana /= vana.coords['gd_prtn_chrg']"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "97ecaa8f-c905-4d02-aad6-dc7ef7c707a8",
-   "metadata": {},
-   "source": [
-    "### Removing the variances of the Vanadium data"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "3d41c537-40b1-4ea7-87b1-f0dc3929ef55",
-   "metadata": {},
-   "source": [
-    "<div class=\"alert alert-warning\">\n",
-    "\n",
-    "**Warning**\n",
-    "    \n",
-    "Heybrock et al. (2023) have shown that Scipp's uncertainty propagation is not suited for broadcast operations\n",
-    "such as normalizing event counts by a scalar value, which is the case when normalizing by Vanadium counts.\n",
-    "These operations are forbidden in recent versions of Scipp.\n",
-    "Until an alternative method is found to satisfactorily track the variances in this workflow,\n",
-    "we remove the variances in the Vanadium data.\n",
-    "The issue is tracked [here](https://github.com/scipp/ess/issues/171).\n",
-    "\n",
-    "</div>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "7ee60cf0-f2f5-426b-89e7-aa137a73b8c5",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "vana.bins.constituents['data'].variances = None"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "31a1525b-c5a2-4164-862d-5e2152d84c83",
-   "metadata": {},
-   "source": [
-    "### Conversion to d-spacing\n",
-    "\n",
-    "Now, we compute d-spacing using the same calibration parameters as before."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "03105e85-f66c-45b4-9b50-adb9192763a1",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "vana_dspacing = powder.to_dspacing_with_calibration(vana, calibration=cal)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "70fd1e76-a16d-4b6d-94d0-fb1284cb26a4",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "vana_dspacing"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "0441b09d-9b7e-45e6-a154-a822a21aebb2",
-   "metadata": {},
-   "source": [
-    "## Inspect d-spacing"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "dd2ef840-a5aa-45dd-aaea-c426dea00a3d",
-   "metadata": {},
-   "source": [
-    "We need to histogram the events in order to normalize our sample data by vanadium.\n",
-    "For consistency, we use these bin edges for both vanadium and the sample data."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ba9ab7db-5752-4130-8cfb-ec00c70feed8",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "d = vana_dspacing.coords['dspacing']\n",
-    "dspacing_edges = sc.linspace('dspacing', d.min().value, d.max().value, 200, unit=d.unit)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "2278a3db-6b79-4421-a1ff-2628226f4166",
-   "metadata": {},
-   "source": [
-    "### All spectra combined"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "84aac5f6-38bd-4246-974b-2a94c2dd80d0",
-   "metadata": {},
-   "source": [
-    "We start simple by combining all spectra using `data.bins.concat('spectrum')`.\n",
-    "Then, we can normalize the same data by vanadium to get a d-spacing distribution.\n",
-    "\n",
-    "**Note that because we removed the variances on the Vanadium data, after the following cell, the standard deviations on the result are underestimated.**"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "5bd287a8-d26e-4134-b1b5-3ed686cca417",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "all_spectra = diffraction.normalize_by_vanadium(\n",
-    "    sample_dspacing.bins.concat('spectrum'),\n",
-    "    vanadium=vana_dspacing.bins.concat('spectrum'),\n",
-    "    edges=dspacing_edges,\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "5ef778f1-e23a-492f-9a08-9a08e889ad83",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "all_spectra.hist(dspacing=dspacing_edges).plot()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "df5f7a70-ac69-47dc-b0c1-1139728d2878",
-   "metadata": {},
-   "source": [
-    "### Group into $2\\theta$ bins"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "fb474efb-692f-4f2d-9243-23ff9319dd76",
-   "metadata": {},
-   "source": [
-    "For a better resolution, we now group the sample and vanadium data into a number of bins in the scattering angle $2\\theta$ (see [here](https://scipp.github.io/scippneutron/user-guide/coordinate-transformations.html))\n",
-    "and normalize each individually."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "a412534e-06e5-4723-9856-37e652dfacf7",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "two_theta = sc.linspace(dim='two_theta', unit='deg', start=25.0, stop=90.0, num=16)\n",
-    "sample_by_two_theta = diffraction.group_by_two_theta(sample_dspacing, edges=two_theta)\n",
-    "vana_by_two_theta = diffraction.group_by_two_theta(vana_dspacing, edges=two_theta)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "0532df2d-a907-40ed-8271-2dc3a71b04ee",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "normalized = diffraction.normalize_by_vanadium(\n",
-    "    sample_by_two_theta, vanadium=vana_by_two_theta, edges=dspacing_edges\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "84c1b8a6-504f-4627-9463-5ff70a6575fd",
-   "metadata": {},
-   "source": [
-    "Histogram the results in order to get a useful binning in the following plots."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "014d6ef4-7905-42bf-8336-df7095de47dc",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "normalized = normalized.hist(dspacing=dspacing_edges)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "460c3a5e-2ab6-4dba-8f78-e4cd126960e8",
-   "metadata": {},
-   "source": [
-    "Now we can inspect the d-spacing distribution as a function of $2\\theta$."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e66a78f7-e91c-4f81-a3ad-e77f6a4c05ae",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "normalized.plot()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "336a7157-1832-4616-b97f-1372fdad3c50",
-   "metadata": {},
-   "source": [
-    "In order to get 1-dimensional plots, we can select some ranges of scattering angles."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "694c8d66-58ea-4645-adca-b33d95f81d23",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "angle = sc.midpoints(normalized.coords['two_theta']).values\n",
-    "results = {\n",
-    "    f'{round(angle[group], 3)} rad': normalized['two_theta', group]\n",
-    "    for group in range(2, 6)\n",
-    "}\n",
-    "sc.plot(results)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "02bd3242-7e90-4279-8cbf-341f2bd37d0c",
-   "metadata": {},
-   "source": [
-    "Or interactively by plotting with a 1d projection."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c41d0965-dfef-40e3-bb9a-5629f51c8b68",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "%matplotlib widget\n",
-    "pp.superplot(normalized)"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.18"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

From 663fb59c9966795f7e79f5727fb388839d942a26 Mon Sep 17 00:00:00 2001
From: Jan-Lukas Wynen <jan-lukas.wynen@ess.eu>
Date: Tue, 28 Nov 2023 11:10:20 +0100
Subject: [PATCH 06/16] Write reduced data to disk

---
 docs/examples/POWGEN_data_reduction.ipynb | 61 ++++++++++++++++++++++-
 src/ess/diffraction/types.py              |  3 ++
 2 files changed, 63 insertions(+), 1 deletion(-)

diff --git a/docs/examples/POWGEN_data_reduction.ipynb b/docs/examples/POWGEN_data_reduction.ipynb
index 8520004..a947ec0 100644
--- a/docs/examples/POWGEN_data_reduction.ipynb
+++ b/docs/examples/POWGEN_data_reduction.ipynb
@@ -169,7 +169,66 @@
    "source": [
     "### Save reduced data to file\n",
     "\n",
-    "TODO"
+    "We ultimately need to write the reduced data to a file.\n",
+    "This could be done with the `result` we computed above.\n",
+    "But then we would have to provide all metadata manually.\n",
+    "Instead, we can use the pipeline to provide this metadata (in this case only the file name) as shown below.\n",
+    "See also the [File output](https://scipp.github.io/sciline/recipes/recipes.html#File-output) docs of Sciline.\n",
+    "\n",
+    "For simplicity we write a simply xye file with 3 columns: $d$-spacing, intensity, standard deviation of intensity."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2ad43ee3-5fe2-4891-b439-4ef1b2204d59",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def save_xye(reduced_data: DspacingHistogram,\n",
+    "             out_filename: OutFilename,\n",
+    ") -> None:\n",
+    "    data = reduced_data.copy(deep=False)\n",
+    "    data.coords['dspacing'] = sc.midpoints(data.coords['dspacing'])\n",
+    "    scn.io.save_xye(out_filename, data, coord='dspacing')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c7f3b9c9-8d6c-4c22-bf56-10f042414d55",
+   "metadata": {},
+   "source": [
+    "Insert a new parameter to set the file name.\n",
+    "This could have been done at the top where the other parameters are defined."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "afdee1fc-abe9-40e8-bae9-cc56e7896fc8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pipeline[OutFilename] = 'reduced.xye'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1eb08353-2bbd-4692-9f8d-7349ddc5d400",
+   "metadata": {},
+   "source": [
+    "And use the pipeline to write the file.\n",
+    "Note that this recomputes the result!"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d35257da-b7a8-47e3-bd90-3bdf98ee539b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pipeline.bind_and_call(save_xye)"
    ]
   },
   {
diff --git a/src/ess/diffraction/types.py b/src/ess/diffraction/types.py
index 3f36ce7..f42242f 100644
--- a/src/ess/diffraction/types.py
+++ b/src/ess/diffraction/types.py
@@ -38,6 +38,9 @@ class Filename(sciline.Scope[RunType, str], str):
     """Filename of a run."""
 
 
+OutFilename = NewType('OutFilename', str)
+"""Filename of the output."""
+
 TwoThetaBins = NewType('TwoThetaBins', sc.Variable)
 """Bin edges for 2theta."""
 

From 5baca32159b461fe200ff9abae218fac8d8967b8 Mon Sep 17 00:00:00 2001
From: Jan-Lukas Wynen <jan-lukas.wynen@ess.eu>
Date: Tue, 28 Nov 2023 11:35:51 +0100
Subject: [PATCH 07/16] Add missing objects to docs

---
 docs/api-reference/index.md      | 23 ++++++++++++++++++-----
 docs/conf.py                     | 15 +++++++++++++++
 src/ess/diffraction/__init__.py  | 26 ++++++++++++++++++++------
 src/ess/diffraction/smoothing.py |  1 +
 4 files changed, 54 insertions(+), 11 deletions(-)

diff --git a/docs/api-reference/index.md b/docs/api-reference/index.md
index a2bf329..250b4a9 100644
--- a/docs/api-reference/index.md
+++ b/docs/api-reference/index.md
@@ -1,14 +1,13 @@
 # API Reference
 
-## Classes
-
+## Module Attributes
 ```{eval-rst}
 .. currentmodule:: ess.diffraction
 
 .. autosummary::
-   :toctree: ../generated/classes
-   :template: class-template.rst
-   :recursive:
+   :toctree: ../generated/attributes
+
+   providers
 ```
 
 ## Top-level functions
@@ -17,6 +16,17 @@
 .. autosummary::
    :toctree: ../generated/functions
    :recursive:
+
+   crop_tof
+   filter_events
+   finalize_histogram
+   group_by_two_theta
+   lowpass
+   merge_all_pixels
+   normalize_by_monitor
+   normalize_by_proton_charge
+   normalize_by_vanadium
+   remove_bad_pulses
 ```
 
 ## Submodules
@@ -27,5 +37,8 @@
    :template: module-template.rst
    :recursive:
 
+   external.powgen
+   powder
    types
+   uncertainty
 ```
\ No newline at end of file
diff --git a/docs/conf.py b/docs/conf.py
index 3e03065..c9c0894 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -200,6 +200,21 @@
 
 doctest_global_setup = '''
 import numpy as np
+
+try:
+    import scipp as sc
+
+    def do_not_plot(*args, **kwargs):
+        pass
+
+    sc.plot = do_not_plot
+    sc.Variable.plot = do_not_plot
+    sc.DataArray.plot = do_not_plot
+    sc.DataGroup.plot = do_not_plot
+    sc.Dataset.plot = do_not_plot
+except ImportError:
+    # Scipp is not needed by docs if it is not installed.
+    pass
 '''
 
 # Using normalize whitespace because many __str__ functions in scipp produce
diff --git a/src/ess/diffraction/__init__.py b/src/ess/diffraction/__init__.py
index d246cfb..35f8de2 100644
--- a/src/ess/diffraction/__init__.py
+++ b/src/ess/diffraction/__init__.py
@@ -7,10 +7,17 @@
 
 import importlib.metadata
 
-from . import filtering, uncertainty
-from .correction import normalize_by_monitor, normalize_by_vanadium
+from . import uncertainty
+from .correction import (
+    normalize_by_monitor,
+    normalize_by_proton_charge,
+    normalize_by_vanadium,
+)
 from .correction import providers as correction_providers
-from .grouping import group_by_two_theta
+from .filtering import crop_tof, filter_events
+from .filtering import providers as filtering_providers
+from .filtering import remove_bad_pulses
+from .grouping import finalize_histogram, group_by_two_theta, merge_all_pixels
 from .grouping import providers as grouping_providers
 from .smoothing import lowpass
 
@@ -22,8 +29,8 @@
 del importlib
 
 providers = (
-    *filtering.providers,
     *correction_providers,
+    *filtering_providers,
     *grouping_providers,
     *uncertainty.providers,
 )
@@ -32,11 +39,18 @@
 These implement basic diffraction data-reduction functionality and need to be
 extended with instrument-specific and sub-technique-specific providers.
 """
-del correction_providers, grouping_providers
+del correction_providers, filtering_providers, grouping_providers
 
 __all__ = [
-    'lowpass',
+    'crop_tof',
+    'filter_events',
+    'finalize_histogram',
     'group_by_two_theta',
+    'lowpass',
+    'merge_all_pixels',
     'normalize_by_monitor',
+    'normalize_by_proton_charge',
     'normalize_by_vanadium',
+    'remove_bad_pulses',
+    'uncertainty',
 ]
diff --git a/src/ess/diffraction/smoothing.py b/src/ess/diffraction/smoothing.py
index 42f3442..b6e8a45 100644
--- a/src/ess/diffraction/smoothing.py
+++ b/src/ess/diffraction/smoothing.py
@@ -70,6 +70,7 @@ def lowpass(
     Examples
     --------
 
+       >>> from ess.diffraction import lowpass
        >>> x = sc.linspace(dim='x', start=1.1, stop=4.0, num=1000, unit='m')
        >>> y = sc.sin(x * sc.scalar(1.0, unit='rad/m'))
        >>> y += sc.sin(x * sc.scalar(400.0, unit='rad/m'))

From 44cb0578e34aa83ad1253f6ea645040f10e74386 Mon Sep 17 00:00:00 2001
From: Jan-Lukas Wynen <jan-lukas.wynen@ess.eu>
Date: Tue, 28 Nov 2023 11:47:11 +0100
Subject: [PATCH 08/16] More docstrings

---
 .../diffraction/external/powgen/beamline.py   |  6 ++++
 src/ess/diffraction/filtering.py              | 35 +++++++++++++++++++
 src/ess/diffraction/grouping.py               | 30 ++++++++++++++--
 3 files changed, 69 insertions(+), 2 deletions(-)

diff --git a/src/ess/diffraction/external/powgen/beamline.py b/src/ess/diffraction/external/powgen/beamline.py
index e0b4248..30206e5 100644
--- a/src/ess/diffraction/external/powgen/beamline.py
+++ b/src/ess/diffraction/external/powgen/beamline.py
@@ -51,6 +51,12 @@ def map_detector_to_spectrum(
 def preprocess_calibration_data(
     data: RawCalibrationData, detector_info: DetectorInfo
 ) -> CalibrationData:
+    """Convert calibration data to a format that can be used by Scipp.
+
+    The raw calibration data is encoded in terms of a `'detector'` coordinate.
+    This needs to be converted to a `'spectrum'` coordinate to align
+    if with sample data.
+    """
     return CalibrationData(map_detector_to_spectrum(data, detector_info=detector_info))
 
 
diff --git a/src/ess/diffraction/filtering.py b/src/ess/diffraction/filtering.py
index c5f8427..f63ce9f 100644
--- a/src/ess/diffraction/filtering.py
+++ b/src/ess/diffraction/filtering.py
@@ -74,12 +74,47 @@ def remove_bad_pulses(
 def crop_tof(
     data: RawData[RunType], tof_range: ValidTofRange
 ) -> TofCroppedData[RunType]:
+    """Remove events outside the specified TOF range.
+
+    Parameters
+    ----------
+    data:
+        Data to be cropped.
+        Expected to have a coordinate called `'tof'`.
+    tof_range:
+        1d, len-2 variable containing the lower and upper bounds for
+        time-of-flight.
+
+    Returns
+    -------
+    :
+        Cropped data.
+    """
     return TofCroppedData[RunType](
         data.bin(tof=tof_range.to(unit=data.coords['tof'].unit))
     )
 
 
 def filter_events(data: TofCroppedData[RunType]) -> FilteredData[RunType]:
+    """Remove bad events.
+
+    Attention
+    ---------
+    This function currently does nothing because it is unclear how to filter
+    events at ESS.
+    In the future, this function will filter out events that
+    cannot be used for analysis.
+
+    Parameters
+    ----------
+    data:
+        Input events to be filtered.
+
+    Returns
+    -------
+    :
+        `data` with bad events removed.
+    """
     # TODO this needs to filter by proton charge once we know how
     return FilteredData[RunType](data)
 
diff --git a/src/ess/diffraction/grouping.py b/src/ess/diffraction/grouping.py
index db5e1ac..c939d17 100644
--- a/src/ess/diffraction/grouping.py
+++ b/src/ess/diffraction/grouping.py
@@ -40,14 +40,40 @@ def group_by_two_theta(
 
 
 def merge_all_pixels(data: DspacingData[RunType]) -> MergedPixels[RunType]:
-    """Combine all pixels (spectra) of the detector."""
+    """Combine all pixels (spectra) of the detector.
+
+    Parameters
+    ----------
+    data:
+        Input data with a `'spectrum'` dimension.
+
+    Returns
+    -------
+    :
+        The input without a `'spectrum'` dimension.
+    """
     return MergedPixels(data.bins.concat('spectrum'))
 
 
 def finalize_histogram(
     data: NormalizedByVanadium, edges: DspacingBins
 ) -> DspacingHistogram:
-    """Finalize the d-spacing histogram."""
+    """Finalize the d-spacing histogram.
+
+    Histograms the input data into the given d-spacing bins.
+
+    Parameters
+    ----------
+    data:
+        Data to be histogrammed.
+    edges:
+        Bin edges in d-spacing.
+
+    Returns
+    -------
+    :
+        Histogrammed data.
+    """
     return DspacingHistogram(data.hist(dspacing=edges))
 
 

From 38727ac3c4d0a5ee47205d39fc1815a94e298c2d Mon Sep 17 00:00:00 2001
From: Jan-Lukas Wynen <jan-lukas.wynen@ess.eu>
Date: Wed, 29 Nov 2023 11:18:48 +0100
Subject: [PATCH 09/16] Add tests of pipeline

---
 tests/diffraction/external/__init__.py        |  0
 tests/diffraction/external/powgen/__init__.py |  0
 .../external/powgen/powgen_reduction_test.py  | 85 +++++++++++++++++++
 3 files changed, 85 insertions(+)
 create mode 100644 tests/diffraction/external/__init__.py
 create mode 100644 tests/diffraction/external/powgen/__init__.py
 create mode 100644 tests/diffraction/external/powgen/powgen_reduction_test.py

diff --git a/tests/diffraction/external/__init__.py b/tests/diffraction/external/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/diffraction/external/powgen/__init__.py b/tests/diffraction/external/powgen/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/diffraction/external/powgen/powgen_reduction_test.py b/tests/diffraction/external/powgen/powgen_reduction_test.py
new file mode 100644
index 0000000..b13071d
--- /dev/null
+++ b/tests/diffraction/external/powgen/powgen_reduction_test.py
@@ -0,0 +1,85 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2023 Scipp contributors (https://github.com/scipp)
+
+import pytest
+import sciline
+import scipp as sc
+
+from ess.diffraction.types import (
+    CalibrationFilename,
+    DspacingBins,
+    DspacingHistogram,
+    Filename,
+    NormalizedByProtonCharge,
+    SampleRun,
+    TwoThetaBins,
+    ValidTofRange,
+    VanadiumRun,
+)
+
+
+@pytest.fixture()
+def providers():
+    from ess import diffraction
+    from ess.diffraction import powder
+    from ess.diffraction.external import powgen
+
+    return [*diffraction.providers, *powder.providers, *powgen.providers]
+
+
+@pytest.fixture()
+def params():
+    return {
+        Filename[SampleRun]: 'PG3_4844_event.zip',
+        Filename[VanadiumRun]: 'PG3_4866_event.zip',
+        CalibrationFilename: 'PG3_FERNS_d4832_2011_08_24.zip',
+        ValidTofRange: sc.array(dims=['tof'], values=[0.0, 16666.67], unit='us'),
+        DspacingBins: sc.linspace('dspacing', 0.0, 2.3434, 200, unit='angstrom'),
+    }
+
+
+def test_can_create_pipeline(providers, params):
+    sciline.Pipeline(providers, params=params)
+
+
+def test_pipeline_can_compute_dspacing_histogram(providers, params):
+    pipeline = sciline.Pipeline(providers, params=params)
+    result = pipeline.compute(DspacingHistogram)
+    assert result.sizes == {
+        'dspacing': len(params[DspacingBins]) - 1,
+    }
+    assert sc.identical(result.coords['dspacing'], params[DspacingBins])
+
+
+def test_workflow_is_deterministic(providers, params):
+    pipeline = sciline.Pipeline(providers, params=params)
+    # This is Sciline's default scheduler, but we want to be explicit here
+    scheduler = sciline.scheduler.DaskScheduler()
+    graph = pipeline.get(DspacingHistogram, scheduler=scheduler)
+    reference = graph.compute().data
+    result = graph.compute().data
+    assert sc.identical(sc.values(result), sc.values(reference))
+
+
+def test_pipeline_can_compute_intermediate_results(providers, params):
+    pipeline = sciline.Pipeline(providers, params=params)
+    result = pipeline.compute(NormalizedByProtonCharge[SampleRun])
+    assert set(result.dims) == {'spectrum', 'tof'}
+
+
+def test_pipeline_group_by_two_theta(providers, params):
+    from ess.diffraction.grouping import group_by_two_theta, merge_all_pixels
+
+    providers.remove(merge_all_pixels)
+    providers.append(group_by_two_theta)
+    params[TwoThetaBins] = sc.linspace(
+        dim='two_theta', unit='deg', start=25.0, stop=90.0, num=16
+    )
+    pipeline = sciline.Pipeline(providers, params=params)
+    result = pipeline.compute(DspacingHistogram)
+    assert result.sizes == {
+        'two_theta': 15,
+        'dspacing': len(params[DspacingBins]) - 1,
+    }
+    assert sc.identical(result.coords['dspacing'], params[DspacingBins])
+    assert sc.allclose(result.coords['two_theta'].to(unit='deg'), params[TwoThetaBins])

From 9c86b506d784fbe98d893f09897ad7180d49c0af Mon Sep 17 00:00:00 2001
From: Jan-Lukas Wynen <jan-lukas.wynen@ess.eu>
Date: Wed, 29 Nov 2023 11:19:26 +0100
Subject: [PATCH 10/16] del internal names

---
 src/ess/diffraction/types.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/ess/diffraction/types.py b/src/ess/diffraction/types.py
index f42242f..3f4fab3 100644
--- a/src/ess/diffraction/types.py
+++ b/src/ess/diffraction/types.py
@@ -113,3 +113,6 @@ class RawDataAndMetadata(sciline.Scope[RunType, sc.DataGroup], sc.DataGroup):
 
 class TofCroppedData(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
     """Raw data cropped to the valid TOF range."""
+
+
+del sc, sciline, NewType, TypeVar

From 9f83d416ff6f85f6f8fc14418a95ae2df02cc307 Mon Sep 17 00:00:00 2001
From: Jan-Lukas Wynen <jan-lukas.wynen@ess.eu>
Date: Tue, 5 Dec 2023 15:58:59 +0100
Subject: [PATCH 11/16] merge -> focus

---
 docs/examples/POWGEN_data_reduction.ipynb |  4 ++--
 src/ess/diffraction/correction.py         |  6 +++---
 src/ess/diffraction/grouping.py           | 10 +++++-----
 src/ess/diffraction/types.py              |  4 ++--
 4 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/docs/examples/POWGEN_data_reduction.ipynb b/docs/examples/POWGEN_data_reduction.ipynb
index a947ec0..6abc6f9 100644
--- a/docs/examples/POWGEN_data_reduction.ipynb
+++ b/docs/examples/POWGEN_data_reduction.ipynb
@@ -298,8 +298,8 @@
    "source": [
     "## Group by scattering angle\n",
     "\n",
-    "The above pipeline merges all instrument pixels to produce a 1d d-spacing curve.\n",
-    "If instead we want to group into $2\\theta$ bins, we can alter the pipeline by replacing the grouping step:"
+    "The above pipeline focuses the data by merging all instrument pixels to produce a 1d d-spacing curve.\n",
+    "If instead we want to group into $2\\theta$ bins, we can alter the pipeline by replacing the focussing step:"
    ]
   },
   {
diff --git a/src/ess/diffraction/correction.py b/src/ess/diffraction/correction.py
index b094a8c..0bffc15 100644
--- a/src/ess/diffraction/correction.py
+++ b/src/ess/diffraction/correction.py
@@ -11,7 +11,7 @@
     AccumulatedProtonCharge,
     DspacingBins,
     FilteredData,
-    MergedPixels,
+    FocussedData,
     NormalizedByProtonCharge,
     NormalizedByVanadium,
     RunType,
@@ -73,8 +73,8 @@ def normalize_by_monitor(
 
 
 def normalize_by_vanadium(
-    data: MergedPixels[SampleRun],
-    vanadium: MergedPixels[VanadiumRun],
+    data: FocussedData[SampleRun],
+    vanadium: FocussedData[VanadiumRun],
     edges: DspacingBins,
 ) -> NormalizedByVanadium:
     """
diff --git a/src/ess/diffraction/grouping.py b/src/ess/diffraction/grouping.py
index c939d17..f56c461 100644
--- a/src/ess/diffraction/grouping.py
+++ b/src/ess/diffraction/grouping.py
@@ -7,7 +7,7 @@
     DspacingBins,
     DspacingData,
     DspacingHistogram,
-    MergedPixels,
+    FocussedData,
     NormalizedByVanadium,
     RunType,
     TwoThetaBins,
@@ -16,7 +16,7 @@
 
 def group_by_two_theta(
     data: DspacingData[RunType], edges: TwoThetaBins
-) -> MergedPixels[RunType]:
+) -> FocussedData[RunType]:
     """
     Group data into two_theta bins.
 
@@ -34,12 +34,12 @@ def group_by_two_theta(
         `data` grouped into two_theta bins.
     """
     out = data.transform_coords('two_theta', graph=beamline.beamline(scatter=True))
-    return MergedPixels[RunType](
+    return FocussedData[RunType](
         out.bin(two_theta=edges.to(unit=out.coords['two_theta'].unit, copy=False))
     )
 
 
-def merge_all_pixels(data: DspacingData[RunType]) -> MergedPixels[RunType]:
+def merge_all_pixels(data: DspacingData[RunType]) -> FocussedData[RunType]:
     """Combine all pixels (spectra) of the detector.
 
     Parameters
@@ -52,7 +52,7 @@ def merge_all_pixels(data: DspacingData[RunType]) -> MergedPixels[RunType]:
     :
         The input without a `'spectrum'` dimension.
     """
-    return MergedPixels(data.bins.concat('spectrum'))
+    return FocussedData(data.bins.concat('spectrum'))
 
 
 def finalize_histogram(
diff --git a/src/ess/diffraction/types.py b/src/ess/diffraction/types.py
index 3f4fab3..729719d 100644
--- a/src/ess/diffraction/types.py
+++ b/src/ess/diffraction/types.py
@@ -79,8 +79,8 @@ class FilteredData(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
     """Raw data without invalid events."""
 
 
-class MergedPixels(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
-    """Intensity vs d-spacing for all detector pixels combined."""
+class FocussedData(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
+    """Intensity vs d-spacing after focussing pixels."""
 
 
 class NormalizedByProtonCharge(sciline.Scope[RunType, sc.DataArray], sc.DataArray):

From a656d3e11be5e828bd4c6584d18516f56fe25b4e Mon Sep 17 00:00:00 2001
From: Jan-Lukas Wynen <jan-lukas.wynen@ess.eu>
Date: Tue, 5 Dec 2023 16:11:27 +0100
Subject: [PATCH 12/16] Explain TwoThetaBins

---
 src/ess/diffraction/types.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/ess/diffraction/types.py b/src/ess/diffraction/types.py
index 729719d..eef585c 100644
--- a/src/ess/diffraction/types.py
+++ b/src/ess/diffraction/types.py
@@ -42,7 +42,11 @@ class Filename(sciline.Scope[RunType, str], str):
 """Filename of the output."""
 
 TwoThetaBins = NewType('TwoThetaBins', sc.Variable)
-"""Bin edges for 2theta."""
+"""Bin edges for grouping in 2theta.
+
+This is used by an alternative focussing step that groups detector
+pixels by scattering angle into bins given by these edges.
+"""
 
 ValidTofRange = NewType('ValidTofRange', sc.Variable)
 """Min and max tof value of the instrument."""

From a7dcc40cacc7a24b58a1a9b81dc29d2b6a1f6eb5 Mon Sep 17 00:00:00 2001
From: Jan-Lukas Wynen <jan-lukas.wynen@ess.eu>
Date: Tue, 5 Dec 2023 16:14:42 +0100
Subject: [PATCH 13/16] Move DetectorInfo into powgen.types

---
 .../diffraction/external/powgen/beamline.py   |  3 ++-
 src/ess/diffraction/external/powgen/data.py   |  2 +-
 src/ess/diffraction/external/powgen/types.py  | 19 +++++++++++++++++++
 src/ess/diffraction/types.py                  |  7 +------
 4 files changed, 23 insertions(+), 8 deletions(-)
 create mode 100644 src/ess/diffraction/external/powgen/types.py

diff --git a/src/ess/diffraction/external/powgen/beamline.py b/src/ess/diffraction/external/powgen/beamline.py
index 30206e5..f889d0a 100644
--- a/src/ess/diffraction/external/powgen/beamline.py
+++ b/src/ess/diffraction/external/powgen/beamline.py
@@ -6,7 +6,8 @@
 
 import scipp as sc
 
-from ...types import CalibrationData, DetectorInfo, RawCalibrationData
+from ...types import CalibrationData, RawCalibrationData
+from .types import DetectorInfo
 
 
 def map_detector_to_spectrum(
diff --git a/src/ess/diffraction/external/powgen/data.py b/src/ess/diffraction/external/powgen/data.py
index 8dc3179..c07ecaa 100644
--- a/src/ess/diffraction/external/powgen/data.py
+++ b/src/ess/diffraction/external/powgen/data.py
@@ -8,7 +8,6 @@
 from ...types import (
     AccumulatedProtonCharge,
     CalibrationFilename,
-    DetectorInfo,
     Filename,
     ProtonCharge,
     RawCalibrationData,
@@ -19,6 +18,7 @@
     SampleRun,
     VanadiumRun,
 )
+from .types import DetectorInfo
 
 _version = '1'
 
diff --git a/src/ess/diffraction/external/powgen/types.py b/src/ess/diffraction/external/powgen/types.py
new file mode 100644
index 0000000..5ad33a5
--- /dev/null
+++ b/src/ess/diffraction/external/powgen/types.py
@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2023 Scipp contributors (https://github.com/scipp)
+
+"""This module defines the domain types used by POWGEN.
+
+The domain types are used to define parameters and to request results from a Sciline
+pipeline.
+"""
+
+from typing import NewType
+
+import scipp as sc
+
+# This is Mantid-specific and can probably be removed when the POWGEN
+# workflow is removed.
+DetectorInfo = NewType('DetectorInfo', sc.Dataset)
+"""Mapping between detector numbers and spectra."""
+
+del sc, NewType
diff --git a/src/ess/diffraction/types.py b/src/ess/diffraction/types.py
index eef585c..db65e34 100644
--- a/src/ess/diffraction/types.py
+++ b/src/ess/diffraction/types.py
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright (c) 2023 Scipp contributors (https://github.com/scipp)
 
-"""This module defines the domain types uses in ess.diffraction.
+"""This module defines the domain types used in ess.diffraction.
 
 The domain types are used to define parameters and to request results from a Sciline
 pipeline.
@@ -61,11 +61,6 @@ class AccumulatedProtonCharge(sciline.Scope[RunType, sc.Variable], sc.Variable):
 CalibrationData = NewType('CalibrationData', sc.Dataset)
 """Detector calibration data."""
 
-# This is Mantid-specific and can probably be removed when the POWGEN
-# workflow is removed.
-DetectorInfo = NewType('DetectorInfo', sc.Dataset)
-"""Mapping between detector numbers and spectra."""
-
 
 class DspacingData(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
     """Data converted to d-spacing."""

From bcdcb3fa7d3eef50c2d0c72e2c1a416a04f6870a Mon Sep 17 00:00:00 2001
From: Jan-Lukas Wynen <jan-lukas.wynen@ess.eu>
Date: Tue, 5 Dec 2023 16:15:37 +0100
Subject: [PATCH 14/16] Plot FilteredData consistently

---
 docs/examples/POWGEN_data_reduction.ipynb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/examples/POWGEN_data_reduction.ipynb b/docs/examples/POWGEN_data_reduction.ipynb
index 6abc6f9..0650b0b 100644
--- a/docs/examples/POWGEN_data_reduction.ipynb
+++ b/docs/examples/POWGEN_data_reduction.ipynb
@@ -285,7 +285,7 @@
    "outputs": [],
    "source": [
     "tof_data = sc.DataGroup(\n",
-    "    sample=results[RawData[SampleRun]].bins.concat('spectrum'),\n",
+    "    sample=results[FilteredData[SampleRun]].bins.concat('spectrum'),\n",
     "    vanadium=results[FilteredData[VanadiumRun]].bins.concat('spectrum'),\n",
     ")\n",
     "tof_data.hist(tof=100).plot()"
@@ -416,7 +416,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.18"
+   "version": "3.10.13"
   }
  },
  "nbformat": 4,

From e2cbb6f8f3bb51bc35568aa2f25a55a7dd271c4c Mon Sep 17 00:00:00 2001
From: Jan-Lukas Wynen <jan-lukas.wynen@ess.eu>
Date: Tue, 5 Dec 2023 16:16:03 +0100
Subject: [PATCH 15/16] Fix typo

---
 src/ess/diffraction/external/powgen/data.py | 6 +++---
 src/ess/diffraction/types.py                | 2 +-
 src/ess/diffraction/uncertainty.py          | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/ess/diffraction/external/powgen/data.py b/src/ess/diffraction/external/powgen/data.py
index c07ecaa..0cfce21 100644
--- a/src/ess/diffraction/external/powgen/data.py
+++ b/src/ess/diffraction/external/powgen/data.py
@@ -13,7 +13,7 @@
     RawCalibrationData,
     RawData,
     RawDataAndMetadata,
-    RawDataWithvariances,
+    RawDataWithVariances,
     RunType,
     SampleRun,
     VanadiumRun,
@@ -125,9 +125,9 @@ def extract_raw_data_sample(dg: RawDataAndMetadata[SampleRun]) -> RawData[Sample
 
 def extract_raw_data_vanadium(
     dg: RawDataAndMetadata[VanadiumRun],
-) -> RawDataWithvariances[VanadiumRun]:
+) -> RawDataWithVariances[VanadiumRun]:
     """Return the events from a loaded data group."""
-    return RawDataWithvariances[VanadiumRun](dg['data'])
+    return RawDataWithVariances[VanadiumRun](dg['data'])
 
 
 def extract_detector_info(dg: RawDataAndMetadata[SampleRun]) -> DetectorInfo:
diff --git a/src/ess/diffraction/types.py b/src/ess/diffraction/types.py
index db65e34..d392b6c 100644
--- a/src/ess/diffraction/types.py
+++ b/src/ess/diffraction/types.py
@@ -102,7 +102,7 @@ class RawData(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
     """Raw data."""
 
 
-class RawDataWithvariances(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
+class RawDataWithVariances(sciline.Scope[RunType, sc.DataArray], sc.DataArray):
     """Raw data that has variances which need special handling."""
 
 
diff --git a/src/ess/diffraction/uncertainty.py b/src/ess/diffraction/uncertainty.py
index 600663f..6031e9b 100644
--- a/src/ess/diffraction/uncertainty.py
+++ b/src/ess/diffraction/uncertainty.py
@@ -2,10 +2,10 @@
 # Copyright (c) 2023 Scipp contributors (https://github.com/scipp)
 """Tools for handling statistical uncertainties."""
 
-from .types import RawData, RawDataWithvariances, VanadiumRun
+from .types import RawData, RawDataWithVariances, VanadiumRun
 
 
-def drop_variances(data: RawDataWithvariances[VanadiumRun]) -> RawData[VanadiumRun]:
+def drop_variances(data: RawDataWithVariances[VanadiumRun]) -> RawData[VanadiumRun]:
     res = data.copy(deep=False)
     if res.bins is not None:
         res.bins.constituents['data'].variances = None

From 87089d1945671766d12f2f2ce6a49279afb48efe Mon Sep 17 00:00:00 2001
From: Jan-Lukas Wynen <jan-lukas.wynen@ess.eu>
Date: Tue, 5 Dec 2023 16:18:33 +0100
Subject: [PATCH 16/16] Do not mention metadata

---
 docs/examples/POWGEN_data_reduction.ipynb | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/examples/POWGEN_data_reduction.ipynb b/docs/examples/POWGEN_data_reduction.ipynb
index 0650b0b..92e9767 100644
--- a/docs/examples/POWGEN_data_reduction.ipynb
+++ b/docs/examples/POWGEN_data_reduction.ipynb
@@ -171,8 +171,7 @@
     "\n",
     "We ultimately need to write the reduced data to a file.\n",
     "This could be done with the `result` we computed above.\n",
-    "But then we would have to provide all metadata manually.\n",
-    "Instead, we can use the pipeline to provide this metadata (in this case only the file name) as shown below.\n",
+    "But we can use the pipeline to provide additional parameters (in this case only the file name) as shown below.\n",
     "See also the [File output](https://scipp.github.io/sciline/recipes/recipes.html#File-output) docs of Sciline.\n",
     "\n",
     "For simplicity we write a simply xye file with 3 columns: $d$-spacing, intensity, standard deviation of intensity."