From ebd8d2d067fb5a93bd68a1a8b54dc873cb09efbc Mon Sep 17 00:00:00 2001 From: ahmedk Date: Tue, 7 Mar 2023 15:36:14 +0100 Subject: [PATCH 1/9] Alter FXE_XAD_JF500K module to te expected 1 JF module coordinate --- extra_data/components.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/extra_data/components.py b/extra_data/components.py index 5e1d50de..265ad9d0 100644 --- a/extra_data/components.py +++ b/extra_data/components.py @@ -1477,7 +1477,7 @@ def __init__(self, data: DataCollection, detector_name=None, modules=None, self._frames_per_entry = self.data[src, self._main_data_key].entry_shape[0] @staticmethod - def _label_dims(arr): + def _label_dims_update_module(arr, detector_name=None): # Label dimensions to match the AGIPD/DSSC/LPD data access ndim_pertrain = arr.ndim if 'trainId' in arr.dims: @@ -1490,6 +1490,12 @@ def _label_dims(arr): }) elif ndim_pertrain == 2: arr = arr.rename({'dim_0': 'cell'}) + + # This detector module source is JNGFR03. Update the module coordinate + # from [3] to [1] + if detector_name == "FXE_XAD_JF500K": + arr['module'] = np.array([1]) + return arr def get_array(self, key, *, fill_value=None, roi=(), astype=None): @@ -1513,7 +1519,7 @@ def get_array(self, key, *, fill_value=None, roi=(), astype=None): input array dtype """ arr = super().get_array(key, fill_value=fill_value, roi=roi, astype=astype) - return self._label_dims(arr) + return self._label_dims_update_module(arr, self.detector_name) def get_dask_array(self, key, fill_value=None, astype=None): """Get a labelled Dask array of detector data @@ -1534,7 +1540,7 @@ def get_dask_array(self, key, fill_value=None, astype=None): input array dtype """ arr = super().get_dask_array(key, fill_value=fill_value, astype=astype) - return self._label_dims(arr) + return self._label_dims_update_module(arr, self.detector_name) def trains(self, require_all=True): """Iterate over trains for detector data. @@ -1552,7 +1558,9 @@ def trains(self, require_all=True): arrays. """ for tid, d in super().trains(require_all=require_all): - yield tid, {k: self._label_dims(a) for (k, a) in d.items()} + yield tid, { + k: self._label_dims_update_module( + a, self.detector_name) for (k, a) in d.items()} def write_virtual_cxi(self, filename, fillvalues=None): """Write a virtual CXI file to access the detector data. From af49073a524524da5e8673b6c7f77bb1aee90c27 Mon Sep 17 00:00:00 2001 From: ahmedk Date: Tue, 14 Mar 2023 10:42:55 +0100 Subject: [PATCH 2/9] revert _label_dims_update_module to _label_dims and introduce st_modno --- extra_data/components.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/extra_data/components.py b/extra_data/components.py index 265ad9d0..802c9ff7 100644 --- a/extra_data/components.py +++ b/extra_data/components.py @@ -1446,6 +1446,9 @@ class JUNGFRAU(MultimodDetectorBase): n_modules: int Number of detector modules in the experiment setup. Default is None, in which case it will be estimated from the available data. + st_modno: int + The module number that the detector starts with. + e.g. FXE_XAD_JF500K/DET/JNGFR03:daqOutput should has st_modno = 3 """ # We appear to have a few different formats for source names: # SPB_IRDA_JNGFR/DET/MODULE_1:daqOutput (e.g. in p 2566, r 61) @@ -1461,9 +1464,19 @@ class JUNGFRAU(MultimodDetectorBase): module_shape = (512, 1024) def __init__(self, data: DataCollection, detector_name=None, modules=None, - *, min_modules=1, n_modules=None): + *, min_modules=1, n_modules=None, st_modno=1): super().__init__(data, detector_name, modules, min_modules=min_modules) + if st_modno: + self.modno_to_source = {} + # Overwrite modno based on given starting module number and update + # source_to_modno and modno_to_source. + for source in self.source_to_modno.keys(): + # JUNGFRAU modono is expected (e.g. extra_geom) to start with 1. + modno = int(self._source_re.search(source)['modno']) - st_modno + 1 + self.source_to_modno[source] = modno + self.modno_to_source[modno] = source + if n_modules is not None: self.n_modules = int(n_modules) else: @@ -1477,7 +1490,7 @@ def __init__(self, data: DataCollection, detector_name=None, modules=None, self._frames_per_entry = self.data[src, self._main_data_key].entry_shape[0] @staticmethod - def _label_dims_update_module(arr, detector_name=None): + def _label_dims(arr): # Label dimensions to match the AGIPD/DSSC/LPD data access ndim_pertrain = arr.ndim if 'trainId' in arr.dims: @@ -1490,12 +1503,6 @@ def _label_dims_update_module(arr, detector_name=None): }) elif ndim_pertrain == 2: arr = arr.rename({'dim_0': 'cell'}) - - # This detector module source is JNGFR03. Update the module coordinate - # from [3] to [1] - if detector_name == "FXE_XAD_JF500K": - arr['module'] = np.array([1]) - return arr def get_array(self, key, *, fill_value=None, roi=(), astype=None): @@ -1519,7 +1526,7 @@ def get_array(self, key, *, fill_value=None, roi=(), astype=None): input array dtype """ arr = super().get_array(key, fill_value=fill_value, roi=roi, astype=astype) - return self._label_dims_update_module(arr, self.detector_name) + return self._label_dims(arr) def get_dask_array(self, key, fill_value=None, astype=None): """Get a labelled Dask array of detector data @@ -1540,7 +1547,7 @@ def get_dask_array(self, key, fill_value=None, astype=None): input array dtype """ arr = super().get_dask_array(key, fill_value=fill_value, astype=astype) - return self._label_dims_update_module(arr, self.detector_name) + return self._label_dims(arr) def trains(self, require_all=True): """Iterate over trains for detector data. @@ -1558,9 +1565,7 @@ def trains(self, require_all=True): arrays. """ for tid, d in super().trains(require_all=require_all): - yield tid, { - k: self._label_dims_update_module( - a, self.detector_name) for (k, a) in d.items()} + yield tid, {k: self._label_dims(a) for (k, a) in d.items()} def write_virtual_cxi(self, filename, fillvalues=None): """Write a virtual CXI file to access the detector data. From 6b54ddf866610bc6180fbd137420ab748f63ebc7 Mon Sep 17 00:00:00 2001 From: ahmedk Date: Tue, 14 Mar 2023 11:21:59 +0100 Subject: [PATCH 3/9] add test for st_modno and add mock_fxe_jungfrau_run --- extra_data/tests/conftest.py | 5 +++++ extra_data/tests/make_examples.py | 18 ++++++++++++++++++ extra_data/tests/test_components.py | 24 ++++++++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/extra_data/tests/conftest.py b/extra_data/tests/conftest.py index 63e5acc4..155de362 100644 --- a/extra_data/tests/conftest.py +++ b/extra_data/tests/conftest.py @@ -141,6 +141,11 @@ def mock_jungfrau_run(): make_examples.make_jungfrau_run(td) yield td +@pytest.fixture(scope='session') +def mock_fxe_jungfrau_run(): + with TemporaryDirectory() as td: + make_examples.make_fxe_jungfrau_run(td) + yield td @pytest.fixture(scope='session') def mock_scs_run(): diff --git a/extra_data/tests/make_examples.py b/extra_data/tests/make_examples.py index 07092794..db1ae74e 100644 --- a/extra_data/tests/make_examples.py +++ b/extra_data/tests/make_examples.py @@ -404,6 +404,24 @@ def make_jungfrau_run(dir_path): JUNGFRAUPower('SPB_IRDA_JF4M/MDL/POWER'), ], ntrains=100, chunksize=1, format_version='1.0') +def make_fxe_jungfrau_run(dir_path): + # Naming based on /gpfs/exfel/exp/FXE/202101/p002478/raw/ + for modno in range(1, 3): + path = osp.join(dir_path, f'RAW-R0012-JNGFR{modno:02}-S00000.h5') + write_file(path, [ + JUNGFRAUModule(f'FXE_XAD_JF1M/DET/JNGFR{modno:02}') + ], ntrains=100, chunksize=1, format_version='1.0') + + path = osp.join(dir_path, f'RAW-R0052-JNGFR03-S00000.h5') + write_file(path, [ + JUNGFRAUModule(f'FXE_XAD_JF500K/DET/JNGFR03') + ], ntrains=100, chunksize=1, format_version='1.0') + + write_file(osp.join(dir_path, f'RAW-R0052-JNGFRCTRL00-S00000.h5'), [ + JUNGFRAUControl('FXE_XAD_JF1M/DET/CONTROL'), + JUNGFRAUControl('FXE_XAD_JF500K/DET/CONTROL'), + ], ntrains=100, chunksize=1, format_version='1.0') + def make_remi_run(dir_path): write_file(osp.join(dir_path, f'CORR-R0210-REMI01-S00000.h5'), [ ReconstructedDLD6('SQS_REMI_DLD6/DET/TOP'), diff --git a/extra_data/tests/test_components.py b/extra_data/tests/test_components.py index bfa71f5f..5fff55c4 100644 --- a/extra_data/tests/test_components.py +++ b/extra_data/tests/test_components.py @@ -235,6 +235,30 @@ def test_get_array_jungfrau(mock_jungfrau_run): assert arr.dtype == np.float32 +def test_jungfraus_st_modno(mock_jungfrau_run, mock_fxe_jungfrau_run): + + # Test SPB_IRDA_JF4M component by setting the st_modno to the default value 1. + run = RunDirectory(mock_jungfrau_run) + jf = JUNGFRAU(run.select_trains(by_index[:2]), st_modno=1) + assert jf.detector_name == 'SPB_IRDA_JF4M' + + arr = jf.get_array('data.adc') + assert np.all(arr['module'] == list(range(1, 9))) + + # Test FXE_XAD_JF500K component with and without setting st_modno to 3. + for st_modno, modno in zip([1, 3], [3, 1]): + run = RunDirectory(mock_fxe_jungfrau_run) + jf = JUNGFRAU( + run.select_trains(by_index[:2]), + detector_name='FXE_XAD_JF500K', + st_modno=st_modno, + ) + assert jf.detector_name == 'FXE_XAD_JF500K' + + arr = jf.get_array('data.adc') + assert np.all(arr['module'] == [modno]) + + def test_get_dask_array(mock_fxe_raw_run): run = RunDirectory(mock_fxe_raw_run) det = LPD1M(run) From 4ace38c02a92a8aa7db6ac363f9865cbf4386565 Mon Sep 17 00:00:00 2001 From: ahmedk Date: Tue, 23 May 2023 14:52:39 +0200 Subject: [PATCH 4/9] replace st_modno name with first_modno --- extra_data/components.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/extra_data/components.py b/extra_data/components.py index 802c9ff7..9497b667 100644 --- a/extra_data/components.py +++ b/extra_data/components.py @@ -1446,9 +1446,9 @@ class JUNGFRAU(MultimodDetectorBase): n_modules: int Number of detector modules in the experiment setup. Default is None, in which case it will be estimated from the available data. - st_modno: int + first_modno: int The module number that the detector starts with. - e.g. FXE_XAD_JF500K/DET/JNGFR03:daqOutput should has st_modno = 3 + e.g. FXE_XAD_JF500K/DET/JNGFR03:daqOutput should has first_modno = 3 """ # We appear to have a few different formats for source names: # SPB_IRDA_JNGFR/DET/MODULE_1:daqOutput (e.g. in p 2566, r 61) @@ -1464,16 +1464,16 @@ class JUNGFRAU(MultimodDetectorBase): module_shape = (512, 1024) def __init__(self, data: DataCollection, detector_name=None, modules=None, - *, min_modules=1, n_modules=None, st_modno=1): + *, min_modules=1, n_modules=None, first_modno=1): super().__init__(data, detector_name, modules, min_modules=min_modules) - if st_modno: + if first_modno: self.modno_to_source = {} # Overwrite modno based on given starting module number and update # source_to_modno and modno_to_source. for source in self.source_to_modno.keys(): # JUNGFRAU modono is expected (e.g. extra_geom) to start with 1. - modno = int(self._source_re.search(source)['modno']) - st_modno + 1 + modno = int(self._source_re.search(source)['modno']) - first_modno + 1 self.source_to_modno[source] = modno self.modno_to_source[modno] = source From 9897575bd59989213d70931f09dde5c67d390841 Mon Sep 17 00:00:00 2001 From: ahmedk Date: Tue, 23 May 2023 14:56:18 +0200 Subject: [PATCH 5/9] improve first_modno description --- extra_data/components.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra_data/components.py b/extra_data/components.py index 9497b667..38b28b16 100644 --- a/extra_data/components.py +++ b/extra_data/components.py @@ -1447,7 +1447,7 @@ class JUNGFRAU(MultimodDetectorBase): Number of detector modules in the experiment setup. Default is None, in which case it will be estimated from the available data. first_modno: int - The module number that the detector starts with. + The module number in the source name for the first detector module. e.g. FXE_XAD_JF500K/DET/JNGFR03:daqOutput should has first_modno = 3 """ # We appear to have a few different formats for source names: From 37cfa0874adac26d2df26f8a4c429ab745111aaa Mon Sep 17 00:00:00 2001 From: ahmedk Date: Tue, 23 May 2023 15:03:22 +0200 Subject: [PATCH 6/9] remove unneeded condition for checking first_modno --- extra_data/components.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/extra_data/components.py b/extra_data/components.py index 38b28b16..b3a86d3d 100644 --- a/extra_data/components.py +++ b/extra_data/components.py @@ -1467,15 +1467,14 @@ def __init__(self, data: DataCollection, detector_name=None, modules=None, *, min_modules=1, n_modules=None, first_modno=1): super().__init__(data, detector_name, modules, min_modules=min_modules) - if first_modno: - self.modno_to_source = {} - # Overwrite modno based on given starting module number and update - # source_to_modno and modno_to_source. - for source in self.source_to_modno.keys(): - # JUNGFRAU modono is expected (e.g. extra_geom) to start with 1. - modno = int(self._source_re.search(source)['modno']) - first_modno + 1 - self.source_to_modno[source] = modno - self.modno_to_source[modno] = source + self.modno_to_source = {} + # Overwrite modno based on given starting module number and update + # source_to_modno and modno_to_source. + for source in self.source_to_modno.keys(): + # JUNGFRAU modono is expected (e.g. extra_geom) to start with 1. + modno = int(self._source_re.search(source)['modno']) - first_modno + 1 + self.source_to_modno[source] = modno + self.modno_to_source[modno] = source if n_modules is not None: self.n_modules = int(n_modules) From 642f2a7ecebcc23006b79800825f2881b56b3beb Mon Sep 17 00:00:00 2001 From: ahmedk Date: Tue, 23 May 2023 16:10:52 +0200 Subject: [PATCH 7/9] replace st_modno to first_modno in the tests --- extra_data/components.py | 2 +- extra_data/tests/test_components.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/extra_data/components.py b/extra_data/components.py index b3a86d3d..bc88b1a4 100644 --- a/extra_data/components.py +++ b/extra_data/components.py @@ -1471,7 +1471,7 @@ def __init__(self, data: DataCollection, detector_name=None, modules=None, # Overwrite modno based on given starting module number and update # source_to_modno and modno_to_source. for source in self.source_to_modno.keys(): - # JUNGFRAU modono is expected (e.g. extra_geom) to start with 1. + # JUNGFRAU modno is expected (e.g. extra_geom) to start with 1. modno = int(self._source_re.search(source)['modno']) - first_modno + 1 self.source_to_modno[source] = modno self.modno_to_source[modno] = source diff --git a/extra_data/tests/test_components.py b/extra_data/tests/test_components.py index 5fff55c4..34375c65 100644 --- a/extra_data/tests/test_components.py +++ b/extra_data/tests/test_components.py @@ -235,23 +235,23 @@ def test_get_array_jungfrau(mock_jungfrau_run): assert arr.dtype == np.float32 -def test_jungfraus_st_modno(mock_jungfrau_run, mock_fxe_jungfrau_run): +def test_jungfraus_first_modno(mock_jungfrau_run, mock_fxe_jungfrau_run): - # Test SPB_IRDA_JF4M component by setting the st_modno to the default value 1. + # Test SPB_IRDA_JF4M component by setting the first_modno to the default value 1. run = RunDirectory(mock_jungfrau_run) - jf = JUNGFRAU(run.select_trains(by_index[:2]), st_modno=1) + jf = JUNGFRAU(run.select_trains(by_index[:2]), first_modno=1) assert jf.detector_name == 'SPB_IRDA_JF4M' arr = jf.get_array('data.adc') assert np.all(arr['module'] == list(range(1, 9))) - # Test FXE_XAD_JF500K component with and without setting st_modno to 3. - for st_modno, modno in zip([1, 3], [3, 1]): + # Test FXE_XAD_JF500K component with and without setting first_modno to 3. + for first_modno, modno in zip([1, 3], [3, 1]): run = RunDirectory(mock_fxe_jungfrau_run) jf = JUNGFRAU( run.select_trains(by_index[:2]), detector_name='FXE_XAD_JF500K', - st_modno=st_modno, + first_modno=first_modno, ) assert jf.detector_name == 'FXE_XAD_JF500K' From 77b88ec9ed1944e746bcd979a917e0f10d45abf6 Mon Sep 17 00:00:00 2001 From: ahmedk Date: Tue, 23 May 2023 17:18:06 +0200 Subject: [PATCH 8/9] fix the self.n_modules based on first_modno --- extra_data/components.py | 2 +- extra_data/tests/test_components.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/extra_data/components.py b/extra_data/components.py index bc88b1a4..84ae1f1c 100644 --- a/extra_data/components.py +++ b/extra_data/components.py @@ -1480,7 +1480,7 @@ def __init__(self, data: DataCollection, detector_name=None, modules=None, self.n_modules = int(n_modules) else: # For JUNGFRAU modules are indexed from 1 - self.n_modules = max(modno for (_, modno) in self._source_matches( + self.n_modules = max(modno - first_modno + 1 for (_, modno) in self._source_matches( data, self.detector_name )) diff --git a/extra_data/tests/test_components.py b/extra_data/tests/test_components.py index 34375c65..536594a6 100644 --- a/extra_data/tests/test_components.py +++ b/extra_data/tests/test_components.py @@ -241,6 +241,7 @@ def test_jungfraus_first_modno(mock_jungfrau_run, mock_fxe_jungfrau_run): run = RunDirectory(mock_jungfrau_run) jf = JUNGFRAU(run.select_trains(by_index[:2]), first_modno=1) assert jf.detector_name == 'SPB_IRDA_JF4M' + assert jf.n_modules == 8 arr = jf.get_array('data.adc') assert np.all(arr['module'] == list(range(1, 9))) @@ -254,6 +255,7 @@ def test_jungfraus_first_modno(mock_jungfrau_run, mock_fxe_jungfrau_run): first_modno=first_modno, ) assert jf.detector_name == 'FXE_XAD_JF500K' + assert jf.n_modules == modno arr = jf.get_array('data.adc') assert np.all(arr['module'] == [modno]) From 97f37d9ca8b40241ddc8a510d01817dc313f780a Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 23 May 2023 17:12:32 +0100 Subject: [PATCH 9/9] Fix in docstring --- extra_data/components.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra_data/components.py b/extra_data/components.py index 84ae1f1c..8ba54fd5 100644 --- a/extra_data/components.py +++ b/extra_data/components.py @@ -1448,7 +1448,7 @@ class JUNGFRAU(MultimodDetectorBase): None, in which case it will be estimated from the available data. first_modno: int The module number in the source name for the first detector module. - e.g. FXE_XAD_JF500K/DET/JNGFR03:daqOutput should has first_modno = 3 + e.g. FXE_XAD_JF500K/DET/JNGFR03:daqOutput should have first_modno = 3 """ # We appear to have a few different formats for source names: # SPB_IRDA_JNGFR/DET/MODULE_1:daqOutput (e.g. in p 2566, r 61)