From 3bbfaaea7c8525bd7d43f5817e3d3983b795cab2 Mon Sep 17 00:00:00 2001 From: Samuel Jones Date: Mon, 27 Jan 2025 15:30:26 +0000 Subject: [PATCH 01/12] Add more generic sans2d rules --- rundetection/ingestion/extracts.py | 6 +- rundetection/rules/common_rules.py | 47 -------- rundetection/rules/factory.py | 13 +-- .../rules/{loq_rules.py => sans_rules.py} | 104 +++++++++++------- test/ingestion/test_extracts.py | 5 +- test/rules/test_common_rules.py | 6 +- test/rules/test_factory.py | 9 +- test/rules/test_loq_rules.py | 66 +++-------- .../specifications/loq_specification.json | 4 +- .../specifications/sans2d_specification.json | 7 +- test/test_specifications.py | 10 +- 11 files changed, 108 insertions(+), 169 deletions(-) rename rundetection/rules/{loq_rules.py => sans_rules.py} (66%) diff --git a/rundetection/ingestion/extracts.py b/rundetection/ingestion/extracts.py index 553af4ac..9f2121d2 100644 --- a/rundetection/ingestion/extracts.py +++ b/rundetection/ingestion/extracts.py @@ -34,7 +34,7 @@ def skip_extract(job_request: JobRequest, _: Any) -> JobRequest: return job_request -def loq_extract(job_request: JobRequest, dataset: Any) -> JobRequest: +def sans_extract(job_request: JobRequest, dataset: Any) -> JobRequest: """ Get the sample details and the cycle strings :param job_request: The job request @@ -173,8 +173,8 @@ def get_extraction_function(instrument: str) -> Callable[[JobRequest, Any], JobR return tosca_extract case "osiris": return osiris_extract - case "loq": - return loq_extract + case "loq" | "sans2d": + return sans_extract case "iris": return iris_extract case _: diff --git a/rundetection/rules/common_rules.py b/rundetection/rules/common_rules.py index 35b18363..7bca8744 100644 --- a/rundetection/rules/common_rules.py +++ b/rundetection/rules/common_rules.py @@ -28,53 +28,6 @@ def verify(self, job_request: JobRequest) -> None: job_request.will_reduce = self._value -class CheckIfScatterSANS(Rule[bool]): - def __init__(self, value: bool): - super().__init__(value) - self.should_be_first = True - - def verify(self, job_request: JobRequest) -> None: - if not job_request.experiment_title.endswith("_SANS/TRANS"): - job_request.will_reduce = False - logger.error("Not a scatter run. Does not have _SANS/TRANS at the end of the experiment title.") - return - # If it has empty or direct in the title assume it is a direct run file instead of a normal scatter. - if ( - "empty" in job_request.experiment_title - or "EMPTY" in job_request.experiment_title - or "direct" in job_request.experiment_title - or "DIRECT" in job_request.experiment_title - ): - job_request.will_reduce = False - logger.error( - "If it is a scatter, contains empty or direct in the title and is assumed to be a scatter " - "for an empty can run." - ) - return - if "{" not in job_request.experiment_title and "}" not in job_request.experiment_title: - job_request.will_reduce = False - logger.error("If it is a scatter, contains {} in format {x}_{y}_SANS/TRANS. or {x}_SANS/TRANS.") - return - - -class SansSliceWavs(Rule[str]): - """ - This rule enables users to set the SliceWavs for each script - """ - - def verify(self, job_request: JobRequest) -> None: - job_request.additional_values["slice_wavs"] = self._value - - -class SansPhiLimits(Rule[str]): - """ - This rule enables users to set the PhiLimits for each script - """ - - def verify(self, job_request: JobRequest) -> None: - job_request.additional_values["phi_limits"] = self._value - - class MolSpecStitchRule(Rule[bool]): """ Enables Tosca, Osiris, and Iris Run stitching diff --git a/rundetection/rules/factory.py b/rundetection/rules/factory.py index 6e74c949..42cf9ca1 100644 --- a/rundetection/rules/factory.py +++ b/rundetection/rules/factory.py @@ -5,15 +5,11 @@ from typing import Any from rundetection.rules.common_rules import ( - CheckIfScatterSANS, EnabledRule, MolSpecStitchRule, - SansPhiLimits, - SansSliceWavs, ) from rundetection.rules.inter_rules import InterStitchRule from rundetection.rules.iris_rules import IrisCalibrationRule, IrisReductionRule -from rundetection.rules.loq_rules import LoqFindFiles, LoqUserFile from rundetection.rules.mari_rules import MariMaskFileRule, MariStitchRule, MariWBVANRule from rundetection.rules.osiris_rules import ( OsirisDefaultGraniteAnalyser, @@ -22,6 +18,7 @@ OsirisReflectionCalibrationRule, ) from rundetection.rules.rule import MissingRuleError, Rule, T +from rundetection.rules.sans_rules import CheckIfScatterSANS, SansFindFiles, SansPhiLimits, SansSliceWavs, SansUserFile def rule_factory(key_: str, value: T) -> Rule[Any]: # noqa: C901, PLR0911, PLR0912 @@ -65,12 +62,12 @@ def rule_factory(key_: str, value: T) -> Rule[Any]: # noqa: C901, PLR0911, PLR0 case "checkifscattersans": if isinstance(value, bool): return CheckIfScatterSANS(value) - case "loqfindfiles": + case "loqfindfiles" | "sansfindfiles": if isinstance(value, bool): - return LoqFindFiles(value) - case "loquserfile": + return SansFindFiles(value) + case "loquserfile" | "sansuserfile": if isinstance(value, str): - return LoqUserFile(value) + return SansUserFile(value) case "sansphilimits": if isinstance(value, str): return SansPhiLimits(value) diff --git a/rundetection/rules/loq_rules.py b/rundetection/rules/sans_rules.py similarity index 66% rename from rundetection/rules/loq_rules.py rename to rundetection/rules/sans_rules.py index fd829670..ba395b81 100644 --- a/rundetection/rules/loq_rules.py +++ b/rundetection/rules/sans_rules.py @@ -1,25 +1,18 @@ -""" -Rules for LOQ -""" - from __future__ import annotations -import logging import re -import typing from dataclasses import dataclass -from pathlib import Path +from typing import TYPE_CHECKING import requests import xmltodict +from rundetection.rules.common_rules import logger from rundetection.rules.rule import Rule -if typing.TYPE_CHECKING: +if TYPE_CHECKING: from rundetection.job_requests import JobRequest -logger = logging.getLogger(__name__) - @dataclass class SansFileData: @@ -28,11 +21,6 @@ class SansFileData: run_number: str -def _extract_run_number_from_filename(filename: str) -> str: - # Assume filename looks like so: LOQ00100002.nxs, then strip. - return filename.split(".")[0].lstrip("LOQ").lstrip("0") - - def _is_sample_transmission_file(sans_file: SansFileData, sample_title: str) -> bool: return sample_title in sans_file.title and sans_file.type == "TRANS" @@ -43,7 +31,7 @@ def _is_sample_direct_file(sans_file: SansFileData) -> bool: def _is_can_scatter_file(sans_file: SansFileData, can_title: str) -> bool: title_contents = re.findall(r"{.*?}", sans_file.title) - return len(title_contents) == 1 and can_title == title_contents[0] and sans_file.type == "SANS/TRANS" + return len(title_contents) == 1 and can_title == title_contents[0] and sans_file.type in {"SANS/TRANS", "SANS"} def _is_can_transmission_file(sans_file: SansFileData, can_title: str) -> bool: @@ -79,24 +67,15 @@ def _find_can_trans_file(sans_files: list[SansFileData], can_title: str) -> Sans return None -def find_path_for_run_number(cycle_path: str, run_number: int) -> Path | None: - # 10 is just a magic number, but we needed an unrealistic value for the maximum - for padding in range(11): - potential_path = Path(f"{cycle_path}/LOQ{str(run_number).zfill(padding)}.nxs") - if potential_path.exists(): - return potential_path - return None - - -def grab_cycle_instrument_index(cycle: str) -> str: +def grab_cycle_instrument_index(cycle: str, instrument: str) -> str: _, cycle_year, cycle_num = cycle.split("_") - url = f"http://data.isis.rl.ac.uk/journals/ndxloq/journal_{cycle_year}_{cycle_num}.xml" + url = f"http://data.isis.rl.ac.uk/journals/ndx{instrument.lower()}/journal_{cycle_year}_{cycle_num}.xml" return requests.get(url, timeout=5).text def create_list_of_files(job_request: JobRequest) -> list[SansFileData]: cycle = job_request.additional_values["cycle_string"] - xml = grab_cycle_instrument_index(cycle=cycle) + xml = grab_cycle_instrument_index(cycle=cycle, instrument=job_request.instrument) cycle_run_info = xmltodict.parse(xml) list_of_files = [] for run_info in cycle_run_info["NXroot"]["NXentry"]: @@ -121,10 +100,10 @@ def _set_transmission_file(job_request: JobRequest, sample_title: str, sans_file if not job_request.additional_values["included_trans_as_scatter"]: trans_file = _find_trans_file(sans_files=sans_files, sample_title=sample_title) trans_run_number = trans_file.run_number if trans_file is not None else None - logger.info("LOQ trans found %s", trans_run_number) + logger.info("%s trans found %s", job_request.instrument, trans_run_number) else: trans_run_number = str(job_request.run_number) - logger.info("LOQ trans set as scatter %s", trans_run_number) + logger.info("%s trans set as scatter %s", job_request.instrument, trans_run_number) if trans_run_number is not None: job_request.additional_values["scatter_transmission"] = trans_run_number @@ -132,24 +111,24 @@ def _set_transmission_file(job_request: JobRequest, sample_title: str, sans_file def _set_can_files(can_title: str | None, job_request: JobRequest, sans_files: list[SansFileData]) -> None: if can_title is not None: can_scatter = _find_can_scatter_file(sans_files=sans_files, can_title=can_title) - logger.info("LOQ can scatter found %s", can_scatter) + logger.info("%s can scatter found %s", job_request.instrument, can_scatter) if can_scatter is not None: job_request.additional_values["can_scatter"] = can_scatter.run_number # If using M4 monitor then can scatter is the transmission if not job_request.additional_values["included_trans_as_scatter"]: can_trans = _find_can_trans_file(sans_files=sans_files, can_title=can_title) - logger.info("LOQ can trans found %s", can_trans) + logger.info("%s can trans found %s", job_request.instrument, can_trans) else: can_trans = can_scatter - logger.info("LOQ can trans set as scatter %s", can_scatter) + logger.info("%s can trans set as scatter %s", job_request.instrument, can_scatter) if can_trans is not None and can_scatter is not None: job_request.additional_values["can_transmission"] = can_trans.run_number def _set_direct_files(job_request: JobRequest, sans_files: list[SansFileData]) -> None: direct_file = _find_direct_file(sans_files=sans_files) - logger.info("LOQ direct files found %s", direct_file) + logger.info("%s direct files found %s", job_request.instrument, direct_file) if direct_file is not None: if "scatter_transmission" in job_request.additional_values: job_request.additional_values["scatter_direct"] = direct_file.run_number @@ -157,21 +136,66 @@ def _set_direct_files(job_request: JobRequest, sans_files: list[SansFileData]) - job_request.additional_values["can_direct"] = direct_file.run_number -class LoqFindFiles(Rule[bool]): +class CheckIfScatterSANS(Rule[bool]): + def __init__(self, value: bool): + super().__init__(value) + self.should_be_first = True + + def verify(self, job_request: JobRequest) -> None: + if not job_request.experiment_title.endswith("_SANS/TRANS") and not job_request.experiment_title.endswith("_SANS"): + job_request.will_reduce = False + logger.error("Not a scatter run. Does not have _SANS or _SANS/TRANS at the end of the experiment title.") + return + # If it has empty or direct in the title assume it is a direct run file instead of a normal scatter. + if ( + "empty" in job_request.experiment_title + or "EMPTY" in job_request.experiment_title + or "direct" in job_request.experiment_title + or "DIRECT" in job_request.experiment_title + ): + job_request.will_reduce = False + logger.error( + "If it is a scatter, contains empty or direct in the title and is assumed to be a scatter " + "for an empty can run." + ) + return + if "{" not in job_request.experiment_title and "}" not in job_request.experiment_title: + job_request.will_reduce = False + logger.error("If it is a scatter, contains {} in format {x}_{y}_SANS/TRANS. or {x}_SANS/TRANS.") + return + + +class SansSliceWavs(Rule[str]): + """ + This rule enables users to set the SliceWavs for each script + """ + def verify(self, job_request: JobRequest) -> None: + job_request.additional_values["slice_wavs"] = self._value + + +class SansPhiLimits(Rule[str]): + """ + This rule enables users to set the PhiLimits for each script + """ + def verify(self, job_request: JobRequest) -> None: + job_request.additional_values["phi_limits"] = self._value + + +class SansFindFiles(Rule[bool]): def __init__(self, value: bool): super().__init__(value) self._should_be_last = True def verify(self, job_request: JobRequest) -> None: title = job_request.experiment_title - logger.info("LOQ title is %s", title) + logger.info("%s title is %s", job_request.instrument, title) # Find all of the "titles" [0] is the scatter, [1] is the background title_parts = re.findall(r"{.*?}", title) sample_title = title_parts[0] - logger.info("LOQ sample title is %s", sample_title) + logger.info("%s sample title is %s", job_request.instrument, sample_title) # If background was defined in the title set can title can_title = title_parts[1] if len(title_parts) > 1 else None - logger.info("LOQ can title is %s from list %s", can_title, title_parts) + logger.info("%s can title is %s from list %s", job_request.instrument, can_title, title_parts) # Get the file lists sans_files = create_list_of_files(job_request) @@ -188,8 +212,8 @@ def verify(self, job_request: JobRequest) -> None: _set_direct_files(job_request, sans_files) -class LoqUserFile(Rule[str]): +class SansUserFile(Rule[str]): def verify(self, job_request: JobRequest) -> None: # If M4 in user file then the transmission and scatter files are the same. job_request.additional_values["included_trans_as_scatter"] = "_M4" in self._value - job_request.additional_values["user_file"] = f"/extras/loq/{self._value}" + job_request.additional_values["user_file"] = f"/extras/{job_request.instrument.lower()}/{self._value}" diff --git a/test/ingestion/test_extracts.py b/test/ingestion/test_extracts.py index 7a1b1ad8..3ac6b845 100644 --- a/test/ingestion/test_extracts.py +++ b/test/ingestion/test_extracts.py @@ -20,7 +20,7 @@ from rundetection.job_requests import JobRequest -@pytest.fixture +@pytest.fixture() def job_request(): """job_request fixture""" return JobRequest( @@ -60,7 +60,8 @@ def test_skip_extract(caplog: LogCaptureFixture): ("mari", "mari_extract"), ("tosca", "tosca_extract"), ("osiris", "osiris_extract"), - ("loq", "loq_extract"), + ("loq", "sans_extract"), + ("sans2d", "sans_extract"), ], ) def test_get_extraction_function(input_value, expected_function_name): diff --git a/test/rules/test_common_rules.py b/test/rules/test_common_rules.py index 6dff4363..86c0a80a 100644 --- a/test/rules/test_common_rules.py +++ b/test/rules/test_common_rules.py @@ -10,15 +10,13 @@ from rundetection.ingestion.ingest import JobRequest from rundetection.rules.common_rules import ( - CheckIfScatterSANS, EnabledRule, - SansPhiLimits, - SansSliceWavs, is_y_within_5_percent_of_x, ) +from rundetection.rules.sans_rules import CheckIfScatterSANS, SansPhiLimits, SansSliceWavs -@pytest.fixture +@pytest.fixture() def job_request(): """ job_request Fixture diff --git a/test/rules/test_factory.py b/test/rules/test_factory.py index 5ec9a2d0..f76b079f 100644 --- a/test/rules/test_factory.py +++ b/test/rules/test_factory.py @@ -8,16 +8,12 @@ import pytest from rundetection.rules.common_rules import ( - CheckIfScatterSANS, EnabledRule, MolSpecStitchRule, - SansPhiLimits, - SansSliceWavs, ) from rundetection.rules.factory import rule_factory from rundetection.rules.inter_rules import InterStitchRule from rundetection.rules.iris_rules import IrisCalibrationRule, IrisReductionRule -from rundetection.rules.loq_rules import LoqFindFiles, LoqUserFile from rundetection.rules.mari_rules import MariMaskFileRule, MariStitchRule, MariWBVANRule from rundetection.rules.osiris_rules import ( OsirisDefaultGraniteAnalyser, @@ -26,6 +22,7 @@ OsirisReflectionCalibrationRule, ) from rundetection.rules.rule import MissingRuleError, Rule +from rundetection.rules.sans_rules import CheckIfScatterSANS, SansFindFiles, SansPhiLimits, SansSliceWavs, SansUserFile def assert_correct_rule(name: str, value: Any, rule_type: type[Rule]): @@ -55,8 +52,8 @@ def assert_correct_rule(name: str, value: Any, rule_type: type[Rule]): ("osirisdefaultgraniteanalyser", True, OsirisDefaultGraniteAnalyser), ("osirisreductionmode", True, OsirisReductionModeRule), ("checkifscattersans", True, CheckIfScatterSANS), - ("loquserfile", "loquserfile.toml", LoqUserFile), - ("loqfindfiles", True, LoqFindFiles), + ("loquserfile", "loquserfile.toml", SansUserFile), + ("loqfindfiles", True, SansFindFiles), ("sansphilimits", "[(1.0, 2.0), (3.0, 4.0)]", SansPhiLimits), ("sansslicewavs", "[2.7, 3.7, 4.7, 5.7, 6.7, 8.7, 10.5]", SansSliceWavs), ("irisreduction", True, IrisReductionRule), diff --git a/test/rules/test_loq_rules.py b/test/rules/test_loq_rules.py index fc01b5e6..e9ee68fe 100644 --- a/test/rules/test_loq_rules.py +++ b/test/rules/test_loq_rules.py @@ -1,15 +1,13 @@ -import tempfile from pathlib import Path from unittest import mock import pytest from rundetection.job_requests import JobRequest -from rundetection.rules.loq_rules import ( - LoqFindFiles, - LoqUserFile, +from rundetection.rules.sans_rules import ( SansFileData, - _extract_run_number_from_filename, + SansFindFiles, + SansUserFile, _find_can_scatter_file, _find_can_trans_file, _find_direct_file, @@ -18,7 +16,6 @@ _is_can_transmission_file, _is_sample_direct_file, _is_sample_transmission_file, - find_path_for_run_number, grab_cycle_instrument_index, strip_excess_files, ) @@ -33,14 +30,6 @@ ] -@pytest.mark.parametrize( - ("filename", "result"), - [("LOQ00100002.nxs", "100002"), ("LOQ123456789.nxs", "123456789"), ("LOQ.nxs", ""), ("LOQ00000.nxs", "")], -) -def test_extract_run_number_from_filename(filename, result): - assert _extract_run_number_from_filename(filename) == result - - @pytest.mark.parametrize( ("sans_file", "sample_title", "result"), [ @@ -117,31 +106,6 @@ def test_can_trans_files(): assert _find_can_trans_file(SANS_FILES, "{Apple}") == SANS_FILES[4] -def test_path_for_run_number_with_some_zeros(): - tempdir = tempfile.mkdtemp() - path = f"{tempdir}/LOQ0012345.nxs" - with Path(path).open("a"): - assert find_path_for_run_number(tempdir, 12345) == Path(path) - - -def test_path_for_run_number_with_no_zeros(): - tempdir = tempfile.mkdtemp() - path = f"{tempdir}/LOQ12345.nxs" - with Path(path).open("a"): - assert find_path_for_run_number(tempdir, 12345) == Path(path) - - -def test_path_for_run_number_too_many_zeros(): - tempdir = tempfile.mkdtemp() - with Path(f"{tempdir}/LOQ00000000000012345.nxs").open("a"): - assert find_path_for_run_number(tempdir, 12345) is None - - -def test_path_for_run_number_doesnt_exist(): - tempdir = tempfile.mkdtemp() - assert find_path_for_run_number(tempdir, 12345) is None - - def test_grab_cycle_instrument_index(): with mock.patch("rundetection.rules.loq_rules.requests") as requests: cycle_index_text = grab_cycle_instrument_index("cycle_24_2") @@ -176,7 +140,7 @@ def test_loq_find_files_verify_no_files_left(): additional_requests=[], ) with mock.patch("rundetection.rules.loq_rules.create_list_of_files", return_value=[]): - loq_find_files = LoqFindFiles(value=True) + loq_find_files = SansFindFiles(value=True) loq_find_files.verify(job_request) assert job_request.will_reduce is False @@ -204,7 +168,7 @@ def test_loq_find_files_verify_some_files_found_but_none_valid(): return_value=[SansFileData("", "", ""), SansFileData("", "", ""), SansFileData("", "", "")], ), ): - loq_find_files = LoqFindFiles(value=True) + loq_find_files = SansFindFiles(value=True) loq_find_files.verify(job_request) assert job_request.will_reduce is True assert job_request.additional_values["run_number"] == 0 @@ -237,7 +201,7 @@ def test_loq_find_files_trans_file_found(): ], ), ): - loq_find_files = LoqFindFiles(value=True) + loq_find_files = SansFindFiles(value=True) loq_find_files.verify(job_request) assert job_request.will_reduce is True assert job_request.additional_values["run_number"] == 5 # noqa: PLR2004 @@ -273,7 +237,7 @@ def test_loq_find_files_can_transmission_file_found(): ], ), ): - loq_find_files = LoqFindFiles(value=True) + loq_find_files = SansFindFiles(value=True) loq_find_files.verify(job_request) assert job_request.will_reduce is True assert job_request.additional_values["run_number"] == 5 # noqa: PLR2004 @@ -308,7 +272,7 @@ def test_loq_find_files_direct_file_found(): ], ), ): - loq_find_files = LoqFindFiles(value=True) + loq_find_files = SansFindFiles(value=True) loq_find_files.verify(job_request) assert job_request.will_reduce is True assert job_request.additional_values["run_number"] == 5 # noqa: PLR2004 @@ -344,7 +308,7 @@ def test_loq_find_files_can_scatter_file_found(): ], ), ): - loq_find_files = LoqFindFiles(value=True) + loq_find_files = SansFindFiles(value=True) loq_find_files.verify(job_request) assert job_request.will_reduce is True assert job_request.additional_values["run_number"] == 5 # noqa: PLR2004 @@ -367,7 +331,7 @@ def test_loq_user_file_m3(): additional_values={}, additional_requests=[], ) - LoqUserFile(value="loq_user_file_M3").verify(job_request) + SansUserFile(value="loq_user_file_M3").verify(job_request) assert job_request.additional_values["user_file"] == "/extras/loq/loq_user_file_M3" assert not job_request.additional_values["included_trans_as_scatter"] assert len(job_request.additional_values) == 2 # noqa: PLR2004 @@ -389,7 +353,7 @@ def test_loq_user_file_m4(): additional_values={}, additional_requests=[], ) - LoqUserFile(value="loq_user_file_M4").verify(job_request) + SansUserFile(value="loq_user_file_M4").verify(job_request) assert job_request.additional_values["user_file"] == "/extras/loq/loq_user_file_M4" assert job_request.additional_values["included_trans_as_scatter"] assert len(job_request.additional_values) == 2 # noqa: PLR2004 @@ -401,18 +365,18 @@ def test_loq_verify_checks_m4(): instrument="", experiment_title="{scatter}_{background}_sans/trans", experiment_number="", - filepath=Path("/path/cycle_24_2/LOQ.nxs"), + filepath=Path("/path/cycle_24_4/LOQ.nxs"), run_start="", run_end="", raw_frames=0, good_frames=0, users="", will_reduce=True, - additional_values={"included_trans_as_scatter": True}, + additional_values={"included_trans_as_scatter": True, "cycle_string": "cycle_24_4"}, additional_requests=[], ) with ( - mock.patch("rundetection.rules.loq_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), + # mock.patch("rundetection.rules.loq_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), mock.patch( "rundetection.rules.loq_rules.strip_excess_files", return_value=[ @@ -423,7 +387,7 @@ def test_loq_verify_checks_m4(): ], ), ): - loq_find_files = LoqFindFiles(value=True) + loq_find_files = SansFindFiles(value=True) loq_find_files.verify(job_request) assert job_request.will_reduce is True assert job_request.additional_values["run_number"] == 5 # noqa: PLR2004 diff --git a/test/test_data/specifications/loq_specification.json b/test/test_data/specifications/loq_specification.json index 1c40fa11..39a4dc71 100644 --- a/test/test_data/specifications/loq_specification.json +++ b/test/test_data/specifications/loq_specification.json @@ -1,8 +1,8 @@ { "enabled": true, "checkifscattersans": true, - "loqfindfiles": true, - "loquserfile": "USER_LOQ_244A_M3_Changer_Loq_MERGED_log.toml", + "sansfindfiles": true, + "sansuserfile": "USER_LOQ_244A_M3_Changer_Loq_MERGED_log.toml", "sansslicewavs": "[2.7, 3.7, 4.7, 5.7, 6.7, 8.7, 10.5]", "sansphilimits": "[(-30, 30), (60, 120)]" } diff --git a/test/test_data/specifications/sans2d_specification.json b/test/test_data/specifications/sans2d_specification.json index c994fef1..a41d21e1 100644 --- a/test/test_data/specifications/sans2d_specification.json +++ b/test/test_data/specifications/sans2d_specification.json @@ -1,3 +1,8 @@ { - "enabled": false + "enabled": true, + "checkifscattersans": true, + "sansfindfiles": true, + "sansuserfile": "USER_SANS2D_244N_12m_5_8mm_M3_Ruzicka_DLS_merged_spot.TOML", + "sansslicewavs": "[1.75, 2.75, 3.75, 4.75, 5.75, 6.75, 8.75, 10.75, 12.5]", + "sansphilimits": "[(-30, 30), (60, 120)]" } \ No newline at end of file diff --git a/test/test_specifications.py b/test/test_specifications.py index e20794ee..56f2c562 100644 --- a/test/test_specifications.py +++ b/test/test_specifications.py @@ -14,12 +14,12 @@ from rundetection.ingestion.ingest import JobRequest from rundetection.rules.common_rules import MolSpecStitchRule from rundetection.rules.iris_rules import IrisCalibrationRule, IrisReductionRule -from rundetection.rules.loq_rules import LoqUserFile from rundetection.rules.mari_rules import MariWBVANRule +from rundetection.rules.sans_rules import SansUserFile from rundetection.specifications import InstrumentSpecification -@pytest.fixture +@pytest.fixture() def job_request(): """ JobRequest fixture @@ -28,7 +28,7 @@ def job_request(): return JobRequest(1, "larmor", "1", "1", Path("/archive/larmor/1/1,nxs"), "start time", "end time", 1, 1, "user") -@pytest.fixture +@pytest.fixture() @patch("rundetection.specifications.InstrumentSpecification._load_rules_from_api") def specification(_) -> InstrumentSpecification: """ @@ -46,7 +46,7 @@ def _set_api_key() -> None: os.environ["FIA_API_API_KEY"] = "shh" -@pytest.fixture +@pytest.fixture() def _working_directory_fix(): # Set dir to repo root for purposes of the test. current_working_directory = Path.cwd() @@ -74,7 +74,7 @@ def test_instrument_specification_load_rules_for_api(requests, specification): requests.get.assert_called_once_with( url="http://localhost:8000/instrument/FOO/specification", headers=headers, timeout=1 ) - assert specification._rules == [MariWBVANRule(100), LoqUserFile("user_file.toml"), MolSpecStitchRule(True)] + assert specification._rules == [MariWBVANRule(100), SansUserFile("user_file.toml"), MolSpecStitchRule(True)] @mock.patch("rundetection.specifications.requests") From 6972e8df6cb594cc322ca17a0b56230aac30b2e2 Mon Sep 17 00:00:00 2001 From: Samuel Jones Date: Mon, 27 Jan 2025 15:38:59 +0000 Subject: [PATCH 02/12] Add sans2d tests and update loq tests to use generic rules --- test/rules/test_loq_rules.py | 36 +-- test/rules/test_sans2d_rules.py | 391 ++++++++++++++++++++++++++++++++ 2 files changed, 409 insertions(+), 18 deletions(-) create mode 100644 test/rules/test_sans2d_rules.py diff --git a/test/rules/test_loq_rules.py b/test/rules/test_loq_rules.py index e9ee68fe..8a58403f 100644 --- a/test/rules/test_loq_rules.py +++ b/test/rules/test_loq_rules.py @@ -107,8 +107,8 @@ def test_can_trans_files(): def test_grab_cycle_instrument_index(): - with mock.patch("rundetection.rules.loq_rules.requests") as requests: - cycle_index_text = grab_cycle_instrument_index("cycle_24_2") + with mock.patch("rundetection.rules.sans_rules.requests") as requests: + cycle_index_text = grab_cycle_instrument_index("cycle_24_2", instrument="LOQ") assert cycle_index_text == requests.get.return_value.text requests.get.assert_called_once_with("http://data.isis.rl.ac.uk/journals/ndxloq/journal_24_2.xml", timeout=5) @@ -139,7 +139,7 @@ def test_loq_find_files_verify_no_files_left(): additional_values={"included_trans_as_scatter": False}, additional_requests=[], ) - with mock.patch("rundetection.rules.loq_rules.create_list_of_files", return_value=[]): + with mock.patch("rundetection.rules.sans_rules.create_list_of_files", return_value=[]): loq_find_files = SansFindFiles(value=True) loq_find_files.verify(job_request) assert job_request.will_reduce is False @@ -162,9 +162,9 @@ def test_loq_find_files_verify_some_files_found_but_none_valid(): additional_requests=[], ) with ( - mock.patch("rundetection.rules.loq_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), + mock.patch("rundetection.rules.sans_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), mock.patch( - "rundetection.rules.loq_rules.strip_excess_files", + "rundetection.rules.sans_rules.strip_excess_files", return_value=[SansFileData("", "", ""), SansFileData("", "", ""), SansFileData("", "", "")], ), ): @@ -191,9 +191,9 @@ def test_loq_find_files_trans_file_found(): additional_requests=[], ) with ( - mock.patch("rundetection.rules.loq_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), + mock.patch("rundetection.rules.sans_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), mock.patch( - "rundetection.rules.loq_rules.strip_excess_files", + "rundetection.rules.sans_rules.strip_excess_files", return_value=[ SansFileData(title="{scatter}", type="TRANS", run_number="1"), SansFileData(title="{background}", type="TRANS", run_number="2"), @@ -226,9 +226,9 @@ def test_loq_find_files_can_transmission_file_found(): additional_requests=[], ) with ( - mock.patch("rundetection.rules.loq_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), + mock.patch("rundetection.rules.sans_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), mock.patch( - "rundetection.rules.loq_rules.strip_excess_files", + "rundetection.rules.sans_rules.strip_excess_files", return_value=[ SansFileData(title="{scatter}", type="TRANS", run_number="1"), SansFileData(title="{background}", type="SANS/TRANS", run_number="2"), @@ -261,9 +261,9 @@ def test_loq_find_files_direct_file_found(): additional_requests=[], ) with ( - mock.patch("rundetection.rules.loq_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), + mock.patch("rundetection.rules.sans_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), mock.patch( - "rundetection.rules.loq_rules.strip_excess_files", + "rundetection.rules.sans_rules.strip_excess_files", return_value=[ SansFileData(title="{scatter}", type="TRANS", run_number="1"), SansFileData(title="{background}", type="SANS/TRANS", run_number="2"), @@ -297,9 +297,9 @@ def test_loq_find_files_can_scatter_file_found(): additional_requests=[], ) with ( - mock.patch("rundetection.rules.loq_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), + mock.patch("rundetection.rules.sans_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), mock.patch( - "rundetection.rules.loq_rules.strip_excess_files", + "rundetection.rules.sans_rules.strip_excess_files", return_value=[ SansFileData(title="{scatter}", type="TRANS", run_number="1"), SansFileData(title="{background}", type="SANS/TRANS", run_number="2"), @@ -318,7 +318,7 @@ def test_loq_find_files_can_scatter_file_found(): def test_loq_user_file_m3(): job_request = JobRequest( run_number=0, - instrument="", + instrument="LOQ", experiment_title="", experiment_number="", filepath=Path(), @@ -340,7 +340,7 @@ def test_loq_user_file_m3(): def test_loq_user_file_m4(): job_request = JobRequest( run_number=0, - instrument="", + instrument="LOQ", experiment_title="", experiment_number="", filepath=Path(), @@ -362,7 +362,7 @@ def test_loq_user_file_m4(): def test_loq_verify_checks_m4(): job_request = JobRequest( run_number=5, - instrument="", + instrument="LOQ", experiment_title="{scatter}_{background}_sans/trans", experiment_number="", filepath=Path("/path/cycle_24_4/LOQ.nxs"), @@ -376,9 +376,9 @@ def test_loq_verify_checks_m4(): additional_requests=[], ) with ( - # mock.patch("rundetection.rules.loq_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), + # mock.patch("rundetection.rules.sans_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), mock.patch( - "rundetection.rules.loq_rules.strip_excess_files", + "rundetection.rules.sans_rules.strip_excess_files", return_value=[ SansFileData(title="{scatter}", type="TRANS", run_number="1"), SansFileData(title="{background}", type="SANS/TRANS", run_number="2"), diff --git a/test/rules/test_sans2d_rules.py b/test/rules/test_sans2d_rules.py new file mode 100644 index 00000000..604c43bd --- /dev/null +++ b/test/rules/test_sans2d_rules.py @@ -0,0 +1,391 @@ +from pathlib import Path +from unittest import mock + +import pytest + +from rundetection.job_requests import JobRequest +from rundetection.rules.sans_rules import ( + SansFileData, + SansFindFiles, + SansUserFile, + _find_can_scatter_file, + _find_can_trans_file, + _find_direct_file, + _find_trans_file, + _is_can_scatter_file, + _is_can_transmission_file, + _is_sample_direct_file, + _is_sample_transmission_file, + grab_cycle_instrument_index, + strip_excess_files, +) + +SANS_FILES = [ + SansFileData(title="{direct/empty beam}", type="TRANS", run_number="-1"), + SansFileData(title="{Banana}", type="SANS", run_number="0"), + SansFileData(title="{Banana}", type="TRANS", run_number="1"), + SansFileData(title="{Apple}", type="SANS", run_number="2"), + SansFileData(title="{Apple}", type="TRANS", run_number="3"), + SansFileData(title="{direct beam}", type="TRANS", run_number="4"), +] + + +@pytest.mark.parametrize( + ("sans_file", "sample_title", "result"), + [ + (SansFileData(title="{Banana}", type="SANS", run_number="0"), "Banana", False), + (SansFileData(title="{Banana}", type="TRANS", run_number="0"), "Banana", True), + (SansFileData(title="{Banana}", type="SANS", run_number="0"), "Banana", False), + (SansFileData(title="{Banana}", type="TRANS", run_number="0"), "Apple", False), + ], +) +def test_is_sample_transmission_file(sans_file, sample_title, result): + assert _is_sample_transmission_file(sans_file, sample_title) == result + + +@pytest.mark.parametrize( + ("sans_file", "result"), + [ + (SansFileData(title="{Banana}", type="TRANS", run_number="0"), False), + (SansFileData(title="{Banana direct}", type="SANS", run_number="0"), False), + (SansFileData(title="{Banana direct}", type="TRANS", run_number="0"), True), + (SansFileData(title="{Banana empty}", type="TRANS", run_number="0"), True), + (SansFileData(title="{Banana direct}", type="SANS", run_number="0"), False), + ], +) +def test_is_sample_direct_file(sans_file, result): + assert _is_sample_direct_file(sans_file) == result + + +@pytest.mark.parametrize( + ("sans_file", "can_title", "result"), + [ + (SansFileData(title="{Banana}", type="SANS", run_number="0"), "{Banana}", True), + (SansFileData(title="{Banana}", type="SANS", run_number="0"), "{Apple}", False), + (SansFileData(title="{Banana}", type="TRANS", run_number="0"), "{Banana}", False), + (SansFileData(title="{Banana}_{}", type="TRANS", run_number="0"), "{Banana}", False), + ], +) +def test_is_can_scatter_file(sans_file, can_title, result): + assert _is_can_scatter_file(sans_file, can_title) == result + + +@pytest.mark.parametrize( + ("sans_file", "can_title", "result"), + [ + (SansFileData(title="{Banana}", type="SANS", run_number="0"), "{Banana}", False), + (SansFileData(title="{Banana}", type="TRANS", run_number="0"), "{Apple}", False), + (SansFileData(title="{Banana}", type="TRANS", run_number="0"), "{Banana}", True), + ], +) +def test_is_can_transmission_file(sans_file, can_title, result): + assert _is_can_transmission_file(sans_file, can_title) == result + + +@pytest.mark.parametrize( + ("sans_files", "sample_title", "expected"), + [(SANS_FILES, "{Apple}", SANS_FILES[4]), (SANS_FILES, "{Banana}", SANS_FILES[2])], +) +def test_find_trans_file_success(sans_files, sample_title, expected): + assert _find_trans_file(sans_files, sample_title) == expected + + +def test_find_trans_file_fail(): + assert _find_trans_file(SANS_FILES, "{Lemmon}") is None + + +def test_find_direct_file(): + assert _find_direct_file(SANS_FILES) == SANS_FILES[-1] + + +def test_find_can_scatter_file(): + assert _find_can_scatter_file(SANS_FILES, "{Apple}") == SANS_FILES[3] + + +def test_can_trans_files(): + assert _find_can_trans_file(SANS_FILES, "{Apple}") == SANS_FILES[4] + + +def test_grab_cycle_instrument_index(): + with mock.patch("rundetection.rules.sans_rules.requests") as requests: + cycle_index_text = grab_cycle_instrument_index("cycle_24_2", instrument="SANS2D") + assert cycle_index_text == requests.get.return_value.text + requests.get.assert_called_once_with("http://data.isis.rl.ac.uk/journals/ndxsans2d/journal_24_2.xml", timeout=5) + + +def test_strip_excess_files(): + files = [ + SansFileData(title="", type="", run_number="0"), + SansFileData(title="", type="", run_number="1"), + SansFileData(title="", type="", run_number="2"), + ] + new_list = strip_excess_files(files, 1) + assert new_list == [SansFileData(title="", type="", run_number="0")] + + +def test_sans2d_find_files_verify_no_files_left(): + job_request = JobRequest( + run_number=0, + instrument="", + experiment_title="{}_{}_sans/trans", + experiment_number="", + filepath=Path(), + run_start="", + run_end="", + raw_frames=0, + good_frames=0, + users="", + will_reduce=True, + additional_values={"included_trans_as_scatter": False}, + additional_requests=[], + ) + with mock.patch("rundetection.rules.sans_rules.create_list_of_files", return_value=[]): + sans_find_file = SansFindFiles(value=True) + sans_find_file.verify(job_request) + assert job_request.will_reduce is False + + +def test_sans2d_find_files_verify_some_files_found_but_none_valid(): + job_request = JobRequest( + run_number=0, + instrument="", + experiment_title="{}_{}_sans/trans", + experiment_number="", + filepath=Path("/path/cycle_24_2/SANS2D000.nxs"), + run_start="", + run_end="", + raw_frames=0, + good_frames=0, + users="", + will_reduce=True, + additional_values={"included_trans_as_scatter": False}, + additional_requests=[], + ) + with ( + mock.patch("rundetection.rules.sans_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), + mock.patch( + "rundetection.rules.sans_rules.strip_excess_files", + return_value=[SansFileData("", "", ""), SansFileData("", "", ""), SansFileData("", "", "")], + ), + ): + sans2d_find_files = SansFindFiles(value=True) + sans2d_find_files.verify(job_request) + assert job_request.will_reduce is True + assert job_request.additional_values["run_number"] == 0 + + +def test_sans2d_find_files_trans_file_found(): + job_request = JobRequest( + run_number=5, + instrument="", + experiment_title="{scatter}_{background}_sans/trans", + experiment_number="", + filepath=Path("/path/cycle_24_2/SANS2D.nxs"), + run_start="", + run_end="", + raw_frames=0, + good_frames=0, + users="", + will_reduce=True, + additional_values={"included_trans_as_scatter": False}, + additional_requests=[], + ) + with ( + mock.patch("rundetection.rules.sans_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), + mock.patch( + "rundetection.rules.sans_rules.strip_excess_files", + return_value=[ + SansFileData(title="{scatter}", type="TRANS", run_number="1"), + SansFileData(title="{background}", type="TRANS", run_number="2"), + SansFileData(title="{direct}", type="SANS/TRANS", run_number="3"), + ], + ), + ): + sans2d_find_files = SansFindFiles(value=True) + sans2d_find_files.verify(job_request) + assert job_request.will_reduce is True + assert job_request.additional_values["run_number"] == 5 # noqa: PLR2004 + assert job_request.additional_values["scatter_transmission"] == "1" + assert "can_scatter" not in job_request.additional_values + + +def test_sans2d_find_files_can_transmission_file_found(): + job_request = JobRequest( + run_number=5, + instrument="", + experiment_title="{scatter}_{background}_sans/trans", + experiment_number="", + filepath=Path("/path/cycle_24_2/SANS2D.nxs"), + run_start="", + run_end="", + raw_frames=0, + good_frames=0, + users="", + will_reduce=True, + additional_values={"included_trans_as_scatter": False}, + additional_requests=[], + ) + with ( + mock.patch("rundetection.rules.sans_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), + mock.patch( + "rundetection.rules.sans_rules.strip_excess_files", + return_value=[ + SansFileData(title="{scatter}", type="TRANS", run_number="1"), + SansFileData(title="{background}", type="SANS/TRANS", run_number="2"), + SansFileData(title="{background}", type="TRANS", run_number="3"), + SansFileData(title="{direct}", type="SANS/TRANS", run_number="4"), + ], + ), + ): + sans2d_find_files = SansFindFiles(value=True) + sans2d_find_files.verify(job_request) + assert job_request.will_reduce is True + assert job_request.additional_values["run_number"] == 5 # noqa: PLR2004 + assert job_request.additional_values["can_transmission"] == "3" + + +def test_sans2d_find_files_direct_file_found(): + job_request = JobRequest( + run_number=5, + instrument="", + experiment_title="{scatter}_{background}_sans/trans", + experiment_number="", + filepath=Path("/path/cycle_24_2/SANS2D000.nxs"), + run_start="", + run_end="", + raw_frames=0, + good_frames=0, + users="", + will_reduce=True, + additional_values={"included_trans_as_scatter": False}, + additional_requests=[], + ) + with ( + mock.patch("rundetection.rules.sans_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), + mock.patch( + "rundetection.rules.sans_rules.strip_excess_files", + return_value=[ + SansFileData(title="{scatter}", type="TRANS", run_number="1"), + SansFileData(title="{background}", type="SANS/TRANS", run_number="2"), + SansFileData(title="{background}", type="TRANS", run_number="3"), + SansFileData(title="{direct}", type="TRANS", run_number="4"), + ], + ), + ): + sans2d_find_files = SansFindFiles(value=True) + sans2d_find_files.verify(job_request) + assert job_request.will_reduce is True + assert job_request.additional_values["run_number"] == 5 # noqa: PLR2004 + assert job_request.additional_values["scatter_direct"] == "4" + assert job_request.additional_values["can_direct"] == "4" + + +def test_sans2d_find_files_can_scatter_file_found(): + job_request = JobRequest( + run_number=5, + instrument="", + experiment_title="{scatter}_{background}_sans/trans", + experiment_number="", + filepath=Path("/path/cycle_24_2/SANS2D.nxs"), + run_start="", + run_end="", + raw_frames=0, + good_frames=0, + users="", + will_reduce=True, + additional_values={"included_trans_as_scatter": False}, + additional_requests=[], + ) + with ( + mock.patch("rundetection.rules.sans_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), + mock.patch( + "rundetection.rules.sans_rules.strip_excess_files", + return_value=[ + SansFileData(title="{scatter}", type="TRANS", run_number="1"), + SansFileData(title="{background}", type="SANS/TRANS", run_number="2"), + SansFileData(title="{background}", type="TRANS", run_number="3"), + SansFileData(title="{direct}", type="SANS/TRANS", run_number="4"), + ], + ), + ): + sans2d_find_files = SansFindFiles(value=True) + sans2d_find_files.verify(job_request) + assert job_request.will_reduce is True + assert job_request.additional_values["run_number"] == 5 # noqa: PLR2004 + assert job_request.additional_values["can_scatter"] == "2" + + +def test_sans2d_user_file_m3(): + job_request = JobRequest( + run_number=0, + instrument="SANS2D", + experiment_title="", + experiment_number="", + filepath=Path(), + run_start="", + run_end="", + raw_frames=0, + good_frames=0, + users="", + will_reduce=True, + additional_values={}, + additional_requests=[], + ) + SansUserFile(value="sans2d_user_file_M3").verify(job_request) + assert job_request.additional_values["user_file"] == "/extras/sans2d/sans2d_user_file_M3" + assert not job_request.additional_values["included_trans_as_scatter"] + assert len(job_request.additional_values) == 2 # noqa: PLR2004 + + +def test_sans2d_user_file_m4(): + job_request = JobRequest( + run_number=0, + instrument="SANS2D", + experiment_title="", + experiment_number="", + filepath=Path(), + run_start="", + run_end="", + raw_frames=0, + good_frames=0, + users="", + will_reduce=True, + additional_values={}, + additional_requests=[], + ) + SansUserFile(value="sans2d_user_file_M4").verify(job_request) + assert job_request.additional_values["user_file"] == "/extras/sans2d/sans2d_user_file_M4" + assert job_request.additional_values["included_trans_as_scatter"] + assert len(job_request.additional_values) == 2 # noqa: PLR2004 + + +def test_sans2d_verify_checks_m4(): + job_request = JobRequest( + run_number=5, + instrument="SANS2D", + experiment_title="{scatter}_{background}_sans/trans", + experiment_number="", + filepath=Path("/path/cycle_24_4/SANS2D.nxs"), + run_start="", + run_end="", + raw_frames=0, + good_frames=0, + users="", + will_reduce=True, + additional_values={"included_trans_as_scatter": True, "cycle_string": "cycle_24_4"}, + additional_requests=[], + ) + with mock.patch("rundetection.rules.sans_rules.strip_excess_files", + return_value=[ + SansFileData(title="{scatter}", type="TRANS", run_number="1"), + SansFileData(title="{background}", type="SANS/TRANS", run_number="2"), + SansFileData(title="{background}", type="TRANS", run_number="3"), + SansFileData(title="{direct}", type="SANS/TRANS", run_number="4"), + ]): + sans2d_find_files = SansFindFiles(value=True) + sans2d_find_files.verify(job_request) + assert job_request.will_reduce is True + assert job_request.additional_values["run_number"] == 5 # noqa: PLR2004 + assert job_request.additional_values["scatter_transmission"] == "5" + assert job_request.additional_values["can_scatter"] == "2" + assert job_request.additional_values["can_transmission"] == "2" From 36db0b3ff6bfab0527c98a290e6662fd26a39e7e Mon Sep 17 00:00:00 2001 From: Samuel Jones Date: Mon, 27 Jan 2025 15:49:51 +0000 Subject: [PATCH 03/12] Add support for mt to rules --- rundetection/rules/sans_rules.py | 4 +++- test/rules/test_sans2d_rules.py | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/rundetection/rules/sans_rules.py b/rundetection/rules/sans_rules.py index ba395b81..ad68390e 100644 --- a/rundetection/rules/sans_rules.py +++ b/rundetection/rules/sans_rules.py @@ -26,7 +26,9 @@ def _is_sample_transmission_file(sans_file: SansFileData, sample_title: str) -> def _is_sample_direct_file(sans_file: SansFileData) -> bool: - return ("direct" in sans_file.title.lower() or "empty" in sans_file.title.lower()) and sans_file.type == "TRANS" + return (("direct" in sans_file.title.lower() or "empty" in sans_file.title.lower() or "mt " in + sans_file.title.lower() or " mt" in sans_file.title.lower() or sans_file.title.lower() == "{mt}") and + sans_file.type=="TRANS") def _is_can_scatter_file(sans_file: SansFileData, can_title: str) -> bool: diff --git a/test/rules/test_sans2d_rules.py b/test/rules/test_sans2d_rules.py index 604c43bd..9c817182 100644 --- a/test/rules/test_sans2d_rules.py +++ b/test/rules/test_sans2d_rules.py @@ -35,7 +35,6 @@ [ (SansFileData(title="{Banana}", type="SANS", run_number="0"), "Banana", False), (SansFileData(title="{Banana}", type="TRANS", run_number="0"), "Banana", True), - (SansFileData(title="{Banana}", type="SANS", run_number="0"), "Banana", False), (SansFileData(title="{Banana}", type="TRANS", run_number="0"), "Apple", False), ], ) @@ -50,7 +49,12 @@ def test_is_sample_transmission_file(sans_file, sample_title, result): (SansFileData(title="{Banana direct}", type="SANS", run_number="0"), False), (SansFileData(title="{Banana direct}", type="TRANS", run_number="0"), True), (SansFileData(title="{Banana empty}", type="TRANS", run_number="0"), True), - (SansFileData(title="{Banana direct}", type="SANS", run_number="0"), False), + (SansFileData(title="{mt beam}", type="TRANS", run_number="0"), True), + (SansFileData(title="{mt}", type="TRANS", run_number="0"), True), + (SansFileData(title="{beam mt}", type="TRANS", run_number="0"), True), + (SansFileData(title="{beam mt cell}", type="TRANS", run_number="0"), True), + (SansFileData(title="{mtcool}", type="TRANS", run_number="0"), False), + (SansFileData(title="{statemt}", type="TRANS", run_number="0"), False), ], ) def test_is_sample_direct_file(sans_file, result): From 2dd7e3aa2c014c71b1724faf652c9894f47c4c2d Mon Sep 17 00:00:00 2001 From: Samuel Jones Date: Mon, 27 Jan 2025 15:52:02 +0000 Subject: [PATCH 04/12] Fix ruff linting --- test/rules/test_loq_rules.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/test/rules/test_loq_rules.py b/test/rules/test_loq_rules.py index 8a58403f..906da99f 100644 --- a/test/rules/test_loq_rules.py +++ b/test/rules/test_loq_rules.py @@ -375,18 +375,13 @@ def test_loq_verify_checks_m4(): additional_values={"included_trans_as_scatter": True, "cycle_string": "cycle_24_4"}, additional_requests=[], ) - with ( - # mock.patch("rundetection.rules.sans_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), - mock.patch( - "rundetection.rules.sans_rules.strip_excess_files", - return_value=[ - SansFileData(title="{scatter}", type="TRANS", run_number="1"), - SansFileData(title="{background}", type="SANS/TRANS", run_number="2"), - SansFileData(title="{background}", type="TRANS", run_number="3"), - SansFileData(title="{direct}", type="SANS/TRANS", run_number="4"), - ], - ), - ): + with mock.patch("rundetection.rules.sans_rules.strip_excess_files", + return_value=[ + SansFileData(title="{scatter}", type="TRANS", run_number="1"), + SansFileData(title="{background}", type="SANS/TRANS", run_number="2"), + SansFileData(title="{background}", type="TRANS", run_number="3"), + SansFileData(title="{direct}", type="SANS/TRANS", run_number="4"), + ]): loq_find_files = SansFindFiles(value=True) loq_find_files.verify(job_request) assert job_request.will_reduce is True From 2f9af105269476df6da62c3f2fcad2359c2d13fa Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 27 Jan 2025 15:52:38 +0000 Subject: [PATCH 05/12] Formatting and linting commit --- rundetection/rules/sans_rules.py | 16 ++++++++++++---- test/ingestion/test_extracts.py | 2 +- test/rules/test_common_rules.py | 2 +- test/rules/test_loq_rules.py | 16 +++++++++------- test/rules/test_sans2d_rules.py | 16 +++++++++------- test/test_specifications.py | 6 +++--- 6 files changed, 35 insertions(+), 23 deletions(-) diff --git a/rundetection/rules/sans_rules.py b/rundetection/rules/sans_rules.py index ad68390e..8561103b 100644 --- a/rundetection/rules/sans_rules.py +++ b/rundetection/rules/sans_rules.py @@ -26,9 +26,13 @@ def _is_sample_transmission_file(sans_file: SansFileData, sample_title: str) -> def _is_sample_direct_file(sans_file: SansFileData) -> bool: - return (("direct" in sans_file.title.lower() or "empty" in sans_file.title.lower() or "mt " in - sans_file.title.lower() or " mt" in sans_file.title.lower() or sans_file.title.lower() == "{mt}") and - sans_file.type=="TRANS") + return ( + "direct" in sans_file.title.lower() + or "empty" in sans_file.title.lower() + or "mt " in sans_file.title.lower() + or " mt" in sans_file.title.lower() + or sans_file.title.lower() == "{mt}" + ) and sans_file.type == "TRANS" def _is_can_scatter_file(sans_file: SansFileData, can_title: str) -> bool: @@ -144,7 +148,9 @@ def __init__(self, value: bool): self.should_be_first = True def verify(self, job_request: JobRequest) -> None: - if not job_request.experiment_title.endswith("_SANS/TRANS") and not job_request.experiment_title.endswith("_SANS"): + if not job_request.experiment_title.endswith("_SANS/TRANS") and not job_request.experiment_title.endswith( + "_SANS" + ): job_request.will_reduce = False logger.error("Not a scatter run. Does not have _SANS or _SANS/TRANS at the end of the experiment title.") return @@ -171,6 +177,7 @@ class SansSliceWavs(Rule[str]): """ This rule enables users to set the SliceWavs for each script """ + def verify(self, job_request: JobRequest) -> None: job_request.additional_values["slice_wavs"] = self._value @@ -179,6 +186,7 @@ class SansPhiLimits(Rule[str]): """ This rule enables users to set the PhiLimits for each script """ + def verify(self, job_request: JobRequest) -> None: job_request.additional_values["phi_limits"] = self._value diff --git a/test/ingestion/test_extracts.py b/test/ingestion/test_extracts.py index 3ac6b845..01fcdb9b 100644 --- a/test/ingestion/test_extracts.py +++ b/test/ingestion/test_extracts.py @@ -20,7 +20,7 @@ from rundetection.job_requests import JobRequest -@pytest.fixture() +@pytest.fixture def job_request(): """job_request fixture""" return JobRequest( diff --git a/test/rules/test_common_rules.py b/test/rules/test_common_rules.py index 86c0a80a..4919b09b 100644 --- a/test/rules/test_common_rules.py +++ b/test/rules/test_common_rules.py @@ -16,7 +16,7 @@ from rundetection.rules.sans_rules import CheckIfScatterSANS, SansPhiLimits, SansSliceWavs -@pytest.fixture() +@pytest.fixture def job_request(): """ job_request Fixture diff --git a/test/rules/test_loq_rules.py b/test/rules/test_loq_rules.py index 906da99f..7d76f656 100644 --- a/test/rules/test_loq_rules.py +++ b/test/rules/test_loq_rules.py @@ -375,13 +375,15 @@ def test_loq_verify_checks_m4(): additional_values={"included_trans_as_scatter": True, "cycle_string": "cycle_24_4"}, additional_requests=[], ) - with mock.patch("rundetection.rules.sans_rules.strip_excess_files", - return_value=[ - SansFileData(title="{scatter}", type="TRANS", run_number="1"), - SansFileData(title="{background}", type="SANS/TRANS", run_number="2"), - SansFileData(title="{background}", type="TRANS", run_number="3"), - SansFileData(title="{direct}", type="SANS/TRANS", run_number="4"), - ]): + with mock.patch( + "rundetection.rules.sans_rules.strip_excess_files", + return_value=[ + SansFileData(title="{scatter}", type="TRANS", run_number="1"), + SansFileData(title="{background}", type="SANS/TRANS", run_number="2"), + SansFileData(title="{background}", type="TRANS", run_number="3"), + SansFileData(title="{direct}", type="SANS/TRANS", run_number="4"), + ], + ): loq_find_files = SansFindFiles(value=True) loq_find_files.verify(job_request) assert job_request.will_reduce is True diff --git a/test/rules/test_sans2d_rules.py b/test/rules/test_sans2d_rules.py index 9c817182..810e719a 100644 --- a/test/rules/test_sans2d_rules.py +++ b/test/rules/test_sans2d_rules.py @@ -379,13 +379,15 @@ def test_sans2d_verify_checks_m4(): additional_values={"included_trans_as_scatter": True, "cycle_string": "cycle_24_4"}, additional_requests=[], ) - with mock.patch("rundetection.rules.sans_rules.strip_excess_files", - return_value=[ - SansFileData(title="{scatter}", type="TRANS", run_number="1"), - SansFileData(title="{background}", type="SANS/TRANS", run_number="2"), - SansFileData(title="{background}", type="TRANS", run_number="3"), - SansFileData(title="{direct}", type="SANS/TRANS", run_number="4"), - ]): + with mock.patch( + "rundetection.rules.sans_rules.strip_excess_files", + return_value=[ + SansFileData(title="{scatter}", type="TRANS", run_number="1"), + SansFileData(title="{background}", type="SANS/TRANS", run_number="2"), + SansFileData(title="{background}", type="TRANS", run_number="3"), + SansFileData(title="{direct}", type="SANS/TRANS", run_number="4"), + ], + ): sans2d_find_files = SansFindFiles(value=True) sans2d_find_files.verify(job_request) assert job_request.will_reduce is True diff --git a/test/test_specifications.py b/test/test_specifications.py index 56f2c562..1369a987 100644 --- a/test/test_specifications.py +++ b/test/test_specifications.py @@ -19,7 +19,7 @@ from rundetection.specifications import InstrumentSpecification -@pytest.fixture() +@pytest.fixture def job_request(): """ JobRequest fixture @@ -28,7 +28,7 @@ def job_request(): return JobRequest(1, "larmor", "1", "1", Path("/archive/larmor/1/1,nxs"), "start time", "end time", 1, 1, "user") -@pytest.fixture() +@pytest.fixture @patch("rundetection.specifications.InstrumentSpecification._load_rules_from_api") def specification(_) -> InstrumentSpecification: """ @@ -46,7 +46,7 @@ def _set_api_key() -> None: os.environ["FIA_API_API_KEY"] = "shh" -@pytest.fixture() +@pytest.fixture def _working_directory_fix(): # Set dir to repo root for purposes of the test. current_working_directory = Path.cwd() From f5e3e8f93d32e376ad9cbaf8aeddb5f6628d3256 Mon Sep 17 00:00:00 2001 From: Samuel Jones Date: Mon, 27 Jan 2025 15:55:14 +0000 Subject: [PATCH 06/12] Fix busted extract test --- test/ingestion/test_extracts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/ingestion/test_extracts.py b/test/ingestion/test_extracts.py index 01fcdb9b..fa5750ff 100644 --- a/test/ingestion/test_extracts.py +++ b/test/ingestion/test_extracts.py @@ -11,9 +11,9 @@ from rundetection.ingestion.extracts import ( get_cycle_string_from_path, get_extraction_function, - loq_extract, mari_extract, osiris_extract, + sans_extract, skip_extract, tosca_extract, ) @@ -239,7 +239,7 @@ def test_osiris_extract_raises_on_bad_frequencies(job_request): osiris_extract(job_request, dataset) -def test_loq_extract(job_request): +def test_sans_extract(job_request): dataset = { "sample": { "thickness": [1.0], @@ -249,7 +249,7 @@ def test_loq_extract(job_request): } } with patch("rundetection.ingestion.extracts.get_cycle_string_from_path", return_value="some string"): - loq_extract(job_request, dataset) + sans_extract(job_request, dataset) assert job_request.additional_values["cycle_string"] == "some string" assert job_request.additional_values["sample_thickness"] == 1.0 From 47d43a980e8f7cf72f7782db378acff183fe5f69 Mon Sep 17 00:00:00 2001 From: Samuel Jones Date: Mon, 27 Jan 2025 15:59:37 +0000 Subject: [PATCH 07/12] Re-add previously removed mock --- test/rules/test_loq_rules.py | 17 +++++++++-------- test/rules/test_sans2d_rules.py | 17 +++++++++-------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/test/rules/test_loq_rules.py b/test/rules/test_loq_rules.py index 7d76f656..224e753a 100644 --- a/test/rules/test_loq_rules.py +++ b/test/rules/test_loq_rules.py @@ -375,14 +375,15 @@ def test_loq_verify_checks_m4(): additional_values={"included_trans_as_scatter": True, "cycle_string": "cycle_24_4"}, additional_requests=[], ) - with mock.patch( - "rundetection.rules.sans_rules.strip_excess_files", - return_value=[ - SansFileData(title="{scatter}", type="TRANS", run_number="1"), - SansFileData(title="{background}", type="SANS/TRANS", run_number="2"), - SansFileData(title="{background}", type="TRANS", run_number="3"), - SansFileData(title="{direct}", type="SANS/TRANS", run_number="4"), - ], + with ( + mock.patch("rundetection.rules.sans_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), + mock.patch("rundetection.rules.sans_rules.strip_excess_files", + return_value=[ + SansFileData(title="{scatter}", type="TRANS", run_number="1"), + SansFileData(title="{background}", type="SANS/TRANS", run_number="2"), + SansFileData(title="{background}", type="TRANS", run_number="3"), + SansFileData(title="{direct}", type="SANS/TRANS", run_number="4"), + ]) ): loq_find_files = SansFindFiles(value=True) loq_find_files.verify(job_request) diff --git a/test/rules/test_sans2d_rules.py b/test/rules/test_sans2d_rules.py index 810e719a..6dac0389 100644 --- a/test/rules/test_sans2d_rules.py +++ b/test/rules/test_sans2d_rules.py @@ -379,14 +379,15 @@ def test_sans2d_verify_checks_m4(): additional_values={"included_trans_as_scatter": True, "cycle_string": "cycle_24_4"}, additional_requests=[], ) - with mock.patch( - "rundetection.rules.sans_rules.strip_excess_files", - return_value=[ - SansFileData(title="{scatter}", type="TRANS", run_number="1"), - SansFileData(title="{background}", type="SANS/TRANS", run_number="2"), - SansFileData(title="{background}", type="TRANS", run_number="3"), - SansFileData(title="{direct}", type="SANS/TRANS", run_number="4"), - ], + with ( + mock.patch("rundetection.rules.sans_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), + mock.patch("rundetection.rules.sans_rules.strip_excess_files", + return_value=[ + SansFileData(title="{scatter}", type="TRANS", run_number="1"), + SansFileData(title="{background}", type="SANS", run_number="2"), + SansFileData(title="{background}", type="TRANS", run_number="3"), + SansFileData(title="{direct}", type="SANS", run_number="4"), + ]) ): sans2d_find_files = SansFindFiles(value=True) sans2d_find_files.verify(job_request) From 5cb041e23a4e9a48c0422138789780e5021587fc Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 27 Jan 2025 16:00:11 +0000 Subject: [PATCH 08/12] Formatting and linting commit --- test/rules/test_loq_rules.py | 16 +++++++++------- test/rules/test_sans2d_rules.py | 16 +++++++++------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/test/rules/test_loq_rules.py b/test/rules/test_loq_rules.py index 224e753a..4f67532e 100644 --- a/test/rules/test_loq_rules.py +++ b/test/rules/test_loq_rules.py @@ -377,13 +377,15 @@ def test_loq_verify_checks_m4(): ) with ( mock.patch("rundetection.rules.sans_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), - mock.patch("rundetection.rules.sans_rules.strip_excess_files", - return_value=[ - SansFileData(title="{scatter}", type="TRANS", run_number="1"), - SansFileData(title="{background}", type="SANS/TRANS", run_number="2"), - SansFileData(title="{background}", type="TRANS", run_number="3"), - SansFileData(title="{direct}", type="SANS/TRANS", run_number="4"), - ]) + mock.patch( + "rundetection.rules.sans_rules.strip_excess_files", + return_value=[ + SansFileData(title="{scatter}", type="TRANS", run_number="1"), + SansFileData(title="{background}", type="SANS/TRANS", run_number="2"), + SansFileData(title="{background}", type="TRANS", run_number="3"), + SansFileData(title="{direct}", type="SANS/TRANS", run_number="4"), + ], + ), ): loq_find_files = SansFindFiles(value=True) loq_find_files.verify(job_request) diff --git a/test/rules/test_sans2d_rules.py b/test/rules/test_sans2d_rules.py index 6dac0389..c3b30a9d 100644 --- a/test/rules/test_sans2d_rules.py +++ b/test/rules/test_sans2d_rules.py @@ -381,13 +381,15 @@ def test_sans2d_verify_checks_m4(): ) with ( mock.patch("rundetection.rules.sans_rules.create_list_of_files", return_value=[SansFileData("", "", "")]), - mock.patch("rundetection.rules.sans_rules.strip_excess_files", - return_value=[ - SansFileData(title="{scatter}", type="TRANS", run_number="1"), - SansFileData(title="{background}", type="SANS", run_number="2"), - SansFileData(title="{background}", type="TRANS", run_number="3"), - SansFileData(title="{direct}", type="SANS", run_number="4"), - ]) + mock.patch( + "rundetection.rules.sans_rules.strip_excess_files", + return_value=[ + SansFileData(title="{scatter}", type="TRANS", run_number="1"), + SansFileData(title="{background}", type="SANS", run_number="2"), + SansFileData(title="{background}", type="TRANS", run_number="3"), + SansFileData(title="{direct}", type="SANS", run_number="4"), + ], + ), ): sans2d_find_files = SansFindFiles(value=True) sans2d_find_files.verify(job_request) From 89282cabf22490c3e627acd8adfdedb6d573c6cc Mon Sep 17 00:00:00 2001 From: Samuel Jones Date: Mon, 27 Jan 2025 16:12:05 +0000 Subject: [PATCH 09/12] Recalibrate one check by reducing duplication --- rundetection/rules/sans_rules.py | 11 ++++------- test/rules/test_common_rules.py | 4 +++- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/rundetection/rules/sans_rules.py b/rundetection/rules/sans_rules.py index 8561103b..8e150de3 100644 --- a/rundetection/rules/sans_rules.py +++ b/rundetection/rules/sans_rules.py @@ -154,13 +154,10 @@ def verify(self, job_request: JobRequest) -> None: job_request.will_reduce = False logger.error("Not a scatter run. Does not have _SANS or _SANS/TRANS at the end of the experiment title.") return - # If it has empty or direct in the title assume it is a direct run file instead of a normal scatter. - if ( - "empty" in job_request.experiment_title - or "EMPTY" in job_request.experiment_title - or "direct" in job_request.experiment_title - or "DIRECT" in job_request.experiment_title - ): + # If it is a direct fix, sans or trans, it should fail, which is why hard coded TRANS as we want to check + # part of the logic not all. + if _is_sample_direct_file(SansFileData(title=job_request.experiment_title, type="TRANS", + run_number=str(job_request.run_number))): job_request.will_reduce = False logger.error( "If it is a scatter, contains empty or direct in the title and is assumed to be a scatter " diff --git a/test/rules/test_common_rules.py b/test/rules/test_common_rules.py index 4919b09b..061e3df8 100644 --- a/test/rules/test_common_rules.py +++ b/test/rules/test_common_rules.py @@ -47,7 +47,7 @@ def test_enabled_rule_when_not_enabled(job_request) -> None: assert job_request.will_reduce is False -@pytest.mark.parametrize("end_of_title", ["_TRANS", "_SANS", "COOL", "_sans/trans"]) +@pytest.mark.parametrize("end_of_title", ["_TRANS", "COOL", "_sans/trans"]) def test_checkifscattersans_verify_raises_for_no_sans_trans(end_of_title) -> None: job_request = mock.MagicMock() job_request.experiment_title = "{fancy chemical}" + end_of_title @@ -60,6 +60,8 @@ def test_checkifscattersans_verify_raises_for_no_sans_trans(end_of_title) -> Non def test_checkifscattersans_verify_raises_for_direct_or_empty_in_title(to_raise) -> None: job_request = mock.MagicMock() job_request.experiment_title = "{fancy chemical " + to_raise + "}_SANS/TRANS" + job_request.will_reduce = True + job_request.run_number = 223312 CheckIfScatterSANS(True).verify(job_request) assert job_request.will_reduce is False From fcbe9ade11c80bc22ab3fb2d3010e2d02ef016d2 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 27 Jan 2025 16:12:49 +0000 Subject: [PATCH 10/12] Formatting and linting commit --- rundetection/rules/sans_rules.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rundetection/rules/sans_rules.py b/rundetection/rules/sans_rules.py index 8e150de3..3f20ad7c 100644 --- a/rundetection/rules/sans_rules.py +++ b/rundetection/rules/sans_rules.py @@ -156,8 +156,9 @@ def verify(self, job_request: JobRequest) -> None: return # If it is a direct fix, sans or trans, it should fail, which is why hard coded TRANS as we want to check # part of the logic not all. - if _is_sample_direct_file(SansFileData(title=job_request.experiment_title, type="TRANS", - run_number=str(job_request.run_number))): + if _is_sample_direct_file( + SansFileData(title=job_request.experiment_title, type="TRANS", run_number=str(job_request.run_number)) + ): job_request.will_reduce = False logger.error( "If it is a scatter, contains empty or direct in the title and is assumed to be a scatter " From bbbf153964b0564b57a4be897b23a367c51a3a18 Mon Sep 17 00:00:00 2001 From: Samuel Jones Date: Tue, 28 Jan 2025 10:34:01 +0000 Subject: [PATCH 11/12] Not a parsable experiment title, and log levels changed --- rundetection/rules/sans_rules.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/rundetection/rules/sans_rules.py b/rundetection/rules/sans_rules.py index 8e150de3..620a8c2a 100644 --- a/rundetection/rules/sans_rules.py +++ b/rundetection/rules/sans_rules.py @@ -152,21 +152,19 @@ def verify(self, job_request: JobRequest) -> None: "_SANS" ): job_request.will_reduce = False - logger.error("Not a scatter run. Does not have _SANS or _SANS/TRANS at the end of the experiment title.") + logger.info("Not a scatter run. Does not have _SANS or _SANS/TRANS at the end of the experiment title.") return # If it is a direct fix, sans or trans, it should fail, which is why hard coded TRANS as we want to check # part of the logic not all. if _is_sample_direct_file(SansFileData(title=job_request.experiment_title, type="TRANS", run_number=str(job_request.run_number))): job_request.will_reduce = False - logger.error( - "If it is a scatter, contains empty or direct in the title and is assumed to be a scatter " - "for an empty can run." - ) + logger.info("File is an empty cell or direct beam scatter run, and should not be processed") return if "{" not in job_request.experiment_title and "}" not in job_request.experiment_title: job_request.will_reduce = False - logger.error("If it is a scatter, contains {} in format {x}_{y}_SANS/TRANS. or {x}_SANS/TRANS.") + logger.info("Not a parsable scatter title, a scatter contains {} in format {x}_{y}_SANS/TRANS. or " + "{x}_SANS/TRANS.") return From 34737ccb438c69e3660ef9655a23d65d5436b90d Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 28 Jan 2025 10:34:56 +0000 Subject: [PATCH 12/12] Formatting and linting commit --- rundetection/rules/sans_rules.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rundetection/rules/sans_rules.py b/rundetection/rules/sans_rules.py index fb5122a5..c1d36d8b 100644 --- a/rundetection/rules/sans_rules.py +++ b/rundetection/rules/sans_rules.py @@ -164,8 +164,9 @@ def verify(self, job_request: JobRequest) -> None: return if "{" not in job_request.experiment_title and "}" not in job_request.experiment_title: job_request.will_reduce = False - logger.info("Not a parsable scatter title, a scatter contains {} in format {x}_{y}_SANS/TRANS. or " - "{x}_SANS/TRANS.") + logger.info( + "Not a parsable scatter title, a scatter contains {} in format {x}_{y}_SANS/TRANS. or {x}_SANS/TRANS." + ) return