From 2eccb68cfc0ad05a084851cd1670cdeca491a833 Mon Sep 17 00:00:00 2001 From: Reece Boston <52183986+rboston628@users.noreply.github.com> Date: Fri, 17 Jan 2025 11:44:51 -0500 Subject: [PATCH] Simple fix to Defect 8554 (#526) * service always loads diffcal mask, use correct name * add hnew helpers, new test of diffcal mask only * changes from code review * simplify maskFromArray * remove * use preexisting add sample log helper * fixup! use preexisting add sample log helper --- .../backend/dao/state/DetectorState.py | 23 +- src/snapred/backend/data/GroceryService.py | 21 ++ src/snapred/backend/recipe/ReductionRecipe.py | 2 +- .../recipe/algorithm/FocusSpectraAlgorithm.py | 3 +- .../recipe/algorithm/GroupedDetectorIDs.py | 3 +- .../algorithm/RemoveSmoothedBackground.py | 3 +- .../backend/service/ReductionService.py | 79 ++--- tests/unit/backend/dao/test_DetectorState.py | 26 ++ .../unit/backend/data/test_GroceryService.py | 40 ++- .../backend/service/test_ReductionService.py | 282 ++++++++++++++---- tests/util/InstaEats.py | 34 ++- tests/util/helpers.py | 108 ++++++- 12 files changed, 516 insertions(+), 108 deletions(-) create mode 100644 tests/unit/backend/dao/test_DetectorState.py diff --git a/src/snapred/backend/dao/state/DetectorState.py b/src/snapred/backend/dao/state/DetectorState.py index 8c2284e45..d141035d8 100644 --- a/src/snapred/backend/dao/state/DetectorState.py +++ b/src/snapred/backend/dao/state/DetectorState.py @@ -1,6 +1,6 @@ from enum import IntEnum from numbers import Number -from typing import Literal, Tuple +from typing import Dict, Literal, Tuple from pydantic import BaseModel, field_validator @@ -26,3 +26,24 @@ def validate_int(cls, v): # e.g. hdf5 returns `int64` v = int(v) return v + + @classmethod + def constructFromLogValues(cls, logValues): + return DetectorState( + arc=(float(logValues["det_arc1"]), float(logValues["det_arc2"])), + lin=(float(logValues["det_lin1"]), float(logValues["det_lin2"])), + wav=float(logValues["BL3:Chop:Skf1:WavelengthUserReq"]), + freq=float(logValues["BL3:Det:TH:BL:Frequency"]), + guideStat=int(logValues["BL3:Mot:OpticsPos:Pos"]), + ) + + def getLogValues(self) -> Dict[str, str]: + return { + "det_lin1": str(self.lin[0]), + "det_lin2": str(self.lin[1]), + "det_arc1": str(self.arc[0]), + "det_arc2": str(self.arc[1]), + "BL3:Chop:Skf1:WavelengthUserReq": str(self.wav), + "BL3:Det:TH:BL:Frequency": str(self.freq), + "BL3:Mot:OpticsPos:Pos": str(self.guideStat), + } diff --git a/src/snapred/backend/data/GroceryService.py b/src/snapred/backend/data/GroceryService.py index 258f83f58..6a1991a83 100644 --- a/src/snapred/backend/data/GroceryService.py +++ b/src/snapred/backend/data/GroceryService.py @@ -513,6 +513,27 @@ def setWorkspaceTag(self, workspaceName: str, logname: str, logvalue: str): else: raise RuntimeError(f"Workspace {workspaceName} does not exist") + def checkPixelMask(self, pixelMask: WorkspaceName) -> bool: + """ + Check if a pixel mask is valid. + :param pixelMask: the name of the MaskWorkspace to check + :type pixelMask: Workspacename + :return: True if pixelMask is a non-trivial MaskWorkspace in the ADS. False otherwise. + :rtype: bool + """ + # A non-trivial mask is a mask that has a non-zero number of masked pixels. + + if not self.mantidSnapper.mtd.doesExist(pixelMask): + return False + else: + mask = self.mantidSnapper.mtd[pixelMask] + if not isinstance(mask, MaskWorkspace): + return False + elif mask.getNumberMasked() == 0: + return False + else: + return True + ## FETCH METHODS """ The fetch methods orchestrate finding data files, loading them into workspaces, diff --git a/src/snapred/backend/recipe/ReductionRecipe.py b/src/snapred/backend/recipe/ReductionRecipe.py index 65da198a2..8bcec04dd 100644 --- a/src/snapred/backend/recipe/ReductionRecipe.py +++ b/src/snapred/backend/recipe/ReductionRecipe.py @@ -63,7 +63,7 @@ def unbagGroceries(self, groceries: Dict[str, Any]): self.groceries = groceries.copy() self.sampleWs = groceries["inputWorkspace"] self.normalizationWs = groceries.get("normalizationWorkspace", "") - self.maskWs = groceries.get("combinedMask", "") + self.maskWs = groceries.get("combinedPixelMask", "") self.groupingWorkspaces = groceries["groupingWorkspaces"] def _cloneWorkspace(self, inputWorkspace: str, outputWorkspace: str) -> str: diff --git a/src/snapred/backend/recipe/algorithm/FocusSpectraAlgorithm.py b/src/snapred/backend/recipe/algorithm/FocusSpectraAlgorithm.py index f7a31f18a..8617c6928 100644 --- a/src/snapred/backend/recipe/algorithm/FocusSpectraAlgorithm.py +++ b/src/snapred/backend/recipe/algorithm/FocusSpectraAlgorithm.py @@ -7,6 +7,7 @@ PythonAlgorithm, WorkspaceUnitValidator, ) +from mantid.dataobjects import GroupingWorkspace from mantid.kernel import Direction, StringMandatoryValidator from snapred.backend.dao.state.PixelGroup import PixelGroup as Ingredients @@ -78,7 +79,7 @@ def validateInputs(self) -> Dict[str, str]: # make sure the input workspace can be reduced by this grouping workspace inWS = self.mantidSnapper.mtd[self.getPropertyValue("InputWorkspace")] groupWS = self.mantidSnapper.mtd[self.getPropertyValue("GroupingWorkspace")] - if "Grouping" not in groupWS.id(): + if not isinstance(groupWS, GroupingWorkspace): errors["GroupingWorkspace"] = "Grouping workspace must be an actual GroupingWorkspace" elif inWS.getNumberHistograms() == len(groupWS.getGroupIDs()): msg = "The data appears to have already been diffraction focused" diff --git a/src/snapred/backend/recipe/algorithm/GroupedDetectorIDs.py b/src/snapred/backend/recipe/algorithm/GroupedDetectorIDs.py index 6fcd03e84..39e1a8aa9 100644 --- a/src/snapred/backend/recipe/algorithm/GroupedDetectorIDs.py +++ b/src/snapred/backend/recipe/algorithm/GroupedDetectorIDs.py @@ -7,6 +7,7 @@ PythonAlgorithm, mtd, ) +from mantid.dataobjects import GroupingWorkspace from mantid.kernel import Direction, ULongLongPropertyWithValue from snapred.meta.pointer import create_pointer @@ -24,7 +25,7 @@ def category(self): def validateInputs(self) -> Dict[str, str]: err = {} focusWS = mtd[self.getPropertyValue("GroupingWorkspace")] - if focusWS.id() != "GroupingWorkspace": + if not isinstance(focusWS, GroupingWorkspace): err["GroupingWorkspace"] = "The workspace must be a GroupingWorkspace" return err diff --git a/src/snapred/backend/recipe/algorithm/RemoveSmoothedBackground.py b/src/snapred/backend/recipe/algorithm/RemoveSmoothedBackground.py index b014e955d..a2fdec31d 100644 --- a/src/snapred/backend/recipe/algorithm/RemoveSmoothedBackground.py +++ b/src/snapred/backend/recipe/algorithm/RemoveSmoothedBackground.py @@ -1,6 +1,7 @@ from typing import Dict, List from mantid.api import ( + IEventWorkspace, IEventWorkspaceProperty, MatrixWorkspaceProperty, PropertyMode, @@ -94,7 +95,7 @@ def unbagGroceries(self): self.inputWorkspaceName = self.getPropertyValue("InputWorkspace") self.outputWorkspaceName = self.getPropertyValue("OutputWorkspace") self.focusWorkspace = self.getPropertyValue("GroupingWorkspace") - self.isEventWs = mtd[self.inputWorkspaceName].id() == "EventWorkspace" + self.isEventWs = isinstance(mtd[self.inputWorkspaceName], IEventWorkspace) def PyExec(self): """ diff --git a/src/snapred/backend/service/ReductionService.py b/src/snapred/backend/service/ReductionService.py index 142e9b172..23189f042 100644 --- a/src/snapred/backend/service/ReductionService.py +++ b/src/snapred/backend/service/ReductionService.py @@ -1,5 +1,4 @@ import json -from collections.abc import Iterable from pathlib import Path from typing import Any, Dict, List, Optional @@ -274,9 +273,7 @@ def loadAllGroupings(self, runNumber: str, useLiteMode: bool) -> Dict[str, Any]: } # WARNING: `WorkspaceName` does not work with `@FromString`! - def prepCombinedMask( - self, runNumber: str, useLiteMode: bool, timestamp: float, pixelMasks: Iterable[WorkspaceName] - ) -> WorkspaceName: + def prepCombinedMask(self, request: ReductionRequest) -> WorkspaceName: """ Combine all of the individual pixel masks for application and final output """ @@ -289,9 +286,42 @@ def prepCombinedMask( ==> TO / FROM mask-dropdown in Reduction panel This MUST be a list of valid `WorkspaceName` (i.e. containing their original `builder` attribute) """ + runNumber = request.runNumber + useLiteMode = request.useLiteMode + timestamp = request.timestamp combinedMask = wng.reductionPixelMask().runNumber(runNumber).timestamp(timestamp).build() + + # if there is a mask associated with the diffcal file, load it here + calVersion = None + if ContinueWarning.Type.MISSING_DIFFRACTION_CALIBRATION not in request.continueFlags: + calVersion = self.dataFactoryService.getLatestApplicableCalibrationVersion(runNumber, useLiteMode) + if calVersion is not None: # WARNING: version may be _zero_! + self.groceryClerk.name("diffcalMaskWorkspace").diffcal_mask(runNumber, calVersion).useLiteMode( + useLiteMode + ).add() + + # if the user specified masks to use, also pull those + residentMasks = {} + for mask in request.pixelMasks: + match mask.tokens("workspaceType"): + case wngt.REDUCTION_PIXEL_MASK: + runNumber, temp_ts = mask.tokens("runNumber", "timestamp") + self.groceryClerk.name(mask).reduction_pixel_mask(runNumber, temp_ts).useLiteMode(useLiteMode).add() + case wngt.REDUCTION_USER_PIXEL_MASK: + numberTag = mask.tokens("numberTag") + residentMasks[mask] = wng.reductionUserPixelMask().numberTag(numberTag).build() + case _: + raise RuntimeError( + f"reduction pixel mask '{mask}' has unexpected workspace-type '{mask.tokens('workspaceType')}'" # noqa: E501 + ) + # Load all pixel masks + allMasks = self.groceryService.fetchGroceryDict( + self.groceryClerk.buildDict(), + **residentMasks, + ) + self.groceryService.fetchCompatiblePixelMask(combinedMask, runNumber, useLiteMode) - for n, mask in enumerate(pixelMasks): + for n, mask in enumerate(allMasks.values()): self.mantidSnapper.BinaryOperateMasks( f"combine from pixel mask {n}...", InputWorkspace1=combinedMask, @@ -369,37 +399,10 @@ def fetchReductionGroceries(self, request: ReductionRequest) -> Dict[str, Any]: request.runNumber, request.useLiteMode ) - # Fetch pixel masks - residentMasks = {} - combinedPixelMask = None - if request.pixelMasks: - for mask in request.pixelMasks: - match mask.tokens("workspaceType"): - case wngt.REDUCTION_PIXEL_MASK: - runNumber, timestamp = mask.tokens("runNumber", "timestamp") - self.groceryClerk.name(mask).reduction_pixel_mask(runNumber, timestamp).useLiteMode( - request.useLiteMode - ).add() - case wngt.REDUCTION_USER_PIXEL_MASK: - numberTag = mask.tokens("numberTag") - residentMasks[mask] = wng.reductionUserPixelMask().numberTag(numberTag).build() - case _: - raise RuntimeError( - f"reduction pixel mask '{mask}' has unexpected workspace-type '{mask.tokens('workspaceType')}'" # noqa: E501 - ) - if calVersion is not None: # WARNING: version may be _zero_! - self.groceryClerk.name("diffcalMaskWorkspace").diffcal_mask(request.runNumber, calVersion).useLiteMode( - request.useLiteMode - ).add() - # Load any non-resident pixel masks - maskGroceries = self.groceryService.fetchGroceryDict( - self.groceryClerk.buildDict(), - **residentMasks, - ) - # combine all of the pixel masks, for application and final output - combinedPixelMask = self.prepCombinedMask( - request.runNumber, request.useLiteMode, request.timestamp, maskGroceries.values() - ) + # Fetch pixel masks -- if nothing is masked, nullify + combinedPixelMask = self.prepCombinedMask(request) + if not self.groceryService.checkPixelMask(combinedPixelMask): + combinedPixelMask = None # gather the input workspace and the diffcal table self.groceryClerk.name("inputWorkspace").neutron(request.runNumber).useLiteMode(request.useLiteMode).add() @@ -415,8 +418,8 @@ def fetchReductionGroceries(self, request: ReductionRequest) -> Dict[str, Any]: ).add() groceries = self.groceryService.fetchGroceryDict( - groceryDict=self.groceryClerk.buildDict(), - **({"combinedPixelMask": combinedPixelMask} if combinedPixelMask else {}), + self.groceryClerk.buildDict(), + **({"combinedPixelMask": combinedPixelMask} if bool(combinedPixelMask) else {}), ) self._markWorkspaceMetadata(request, groceries["inputWorkspace"]) diff --git a/tests/unit/backend/dao/test_DetectorState.py b/tests/unit/backend/dao/test_DetectorState.py new file mode 100644 index 000000000..77eece397 --- /dev/null +++ b/tests/unit/backend/dao/test_DetectorState.py @@ -0,0 +1,26 @@ +# note: this runs the same checks as the calibrant_samples_script CIS test +import unittest + +from snapred.backend.dao.state.DetectorState import DetectorState + + +class TestDetectorstate(unittest.TestCase): + def test_getLogValues(self): + exp = { + "det_lin1": "1.0", + "det_lin2": "1.1", + "det_arc1": "2.2", + "det_arc2": "2.3", + "BL3:Chop:Skf1:WavelengthUserReq": "3.4", + "BL3:Det:TH:BL:Frequency": "4.5", + "BL3:Mot:OpticsPos:Pos": "2", + } + detectorState = DetectorState.constructFromLogValues(exp) + assert detectorState.arc[0] == float(exp["det_arc1"]) + assert detectorState.arc[1] == float(exp["det_arc2"]) + assert detectorState.lin[0] == float(exp["det_lin1"]) + assert detectorState.lin[1] == float(exp["det_lin2"]) + assert detectorState.wav == float(exp["BL3:Chop:Skf1:WavelengthUserReq"]) + assert detectorState.freq == float(exp["BL3:Det:TH:BL:Frequency"]) + assert detectorState.guideStat == float(exp["BL3:Mot:OpticsPos:Pos"]) + assert exp == detectorState.getLogValues() diff --git a/tests/unit/backend/data/test_GroceryService.py b/tests/unit/backend/data/test_GroceryService.py index 6ad8b7991..8693d70e5 100644 --- a/tests/unit/backend/data/test_GroceryService.py +++ b/tests/unit/backend/data/test_GroceryService.py @@ -13,12 +13,15 @@ from mantid.kernel import V3D, Quat from mantid.simpleapi import ( CloneWorkspace, + CreateSampleWorkspace, CreateWorkspace, DeleteWorkspace, + ExtractMask, GenerateTableWorkspaceFromListOfDict, GroupWorkspaces, LoadEmptyInstrument, LoadInstrument, + MaskDetectors, SaveDiffCal, SaveNexus, SaveNexusProcessed, @@ -174,7 +177,7 @@ def mockIndexer(self, root=None, calType=None): def clearoutWorkspaces(self) -> None: """Delete the workspaces created by loading""" for ws in mtd.getObjectNames(): - if ws not in self.excludeAtTeardown: + if ws not in self.excludeAtTeardown and ws in mtd: DeleteWorkspace(ws) def tearDown(self): @@ -190,7 +193,8 @@ def tearDownClass(cls): and remove the test file. """ for ws in mtd.getObjectNames(): - DeleteWorkspace(ws) + if ws in mtd: + DeleteWorkspace(ws) os.remove(cls.sampleWSFilePath) os.remove(cls.sampleDiffCalFilePath) os.remove(cls.sampleTarWsFilePath) @@ -1893,3 +1897,35 @@ def test_lookupDiffcalTableWorkspaceName_missing_workspace_in_record(self): with pytest.raises(RuntimeError, match=r".*Could not find diffcal table in record*"): self.instance.lookupDiffcalTableWorkspaceName(runNumber, True, version) + + def test_checkPixelMask(self): + # raises an error if workspace not in ADS + nonexistent = mtd.unique_name(prefix="_mask_check_") + assert not mtd.doesExist(nonexistent) + assert not self.instance.checkPixelMask(nonexistent) + + # raises an error if the workspace is not a MaskWorkspace + notamask = mtd.unique_name(prefix="_mask_check_") + CreateSampleWorkspace( + OutputWorkspace=notamask, + NumBanks=1, + BankPixelWidth=1, + ) + assert mtd.doesExist(notamask) + assert not isinstance(mtd[notamask], MaskWorkspace) + assert not self.instance.checkPixelMask(notamask) + + # return False if nothing is masked + emptymask = mtd.unique_name(prefix="_mask_check_") + ExtractMask(InputWorkspace=notamask, OutputWorkspace=emptymask) + assert isinstance(mtd[emptymask], MaskWorkspace) + assert mtd[emptymask].getNumberMasked() == 0 + assert not self.instance.checkPixelMask(emptymask) + + # return True if something is masked + nonemptymask = mtd.unique_name(prefix="_mask_check_") + MaskDetectors(Workspace=notamask, WorkspaceIndexList=[0]) + ExtractMask(InputWorkspace=notamask, OutputWorkspace=nonemptymask) + assert isinstance(mtd[nonemptymask], MaskWorkspace) + assert mtd[nonemptymask].getNumberMasked() != 0 + assert self.instance.checkPixelMask(nonemptymask) diff --git a/tests/unit/backend/service/test_ReductionService.py b/tests/unit/backend/service/test_ReductionService.py index 10db2143d..ca298721e 100644 --- a/tests/unit/backend/service/test_ReductionService.py +++ b/tests/unit/backend/service/test_ReductionService.py @@ -10,9 +10,11 @@ DeleteWorkspace, mtd, ) +from mantid.testing import assert_almost_equal as wksp_almost_equal from util.helpers import ( arrayFromMask, createCompatibleMask, + maskFromArray, ) from util.InstaEats import InstaEats from util.SculleryBoy import SculleryBoy @@ -134,6 +136,37 @@ def test_fetchReductionGroceries(self): assert "diffcalWorkspace" in res assert "normalizationWorkspace" in res + def test_fetchReductionGroceries_use_mask(self): + """ + Check that this properly handles using the reduction mask. + This actually sets if the mask workspace inside the RECIPE is set. + - when a mask is created by prepCombineMask and is non-empty, then should be sent to recipe + - otherwise, there should be no mask sent to the recipe + """ + from snapred.backend.recipe.ReductionRecipe import ReductionRecipe + + self.instance.dataFactoryService.getLatestApplicableCalibrationVersion = mock.Mock(return_value=1) + self.instance.dataFactoryService.getLatestApplicableNormalizationVersion = mock.Mock(return_value=1) + self.instance._markWorkspaceMetadata = mock.Mock() + self.instance.prepCombinedMask = mock.Mock(return_value=mock.sentinel.mask) + self.request.continueFlags = ContinueWarning.Type.UNSET + + rx = ReductionRecipe() + + # situation where mask is true -- ensure mask is set + self.instance.groceryService.checkPixelMask = mock.Mock(return_value=True) + res = self.instance.fetchReductionGroceries(self.request) + res["groupingWorkspaces"] = [mock.sentinel.groupingWS] + rx.unbagGroceries(res) + assert rx.maskWs == mock.sentinel.mask + + # change mask to be false -- make sure unused + self.instance.groceryService.checkPixelMask.return_value = False + res = self.instance.fetchReductionGroceries(self.request) + res["groupingWorkspaces"] = [mock.sentinel.groupingWS] + rx.unbagGroceries(res) + assert rx.maskWs == "" + @mock.patch(thisService + "ReductionRecipe") def test_reduction(self, mockReductionRecipe): mockReductionRecipe.return_value = mock.Mock() @@ -652,7 +685,10 @@ def _createCompatibleMask(maskWSName, runNumber): # teardown... pass - def test_prepCombinedMask(self): + def test_prepCombinedMask_correct(self): + """ + Check that prepCombinedMask correctly combines pixel masks + """ masks = [self.maskWS1, self.maskWS2] maskArrays = [arrayFromMask(mask) for mask in masks] @@ -660,24 +696,38 @@ def test_prepCombinedMask(self): # otherwise `prepCombinedMask` might overwrite one of the # sample mask workspaces! timestamp = self.service.getUniqueTimestamp() - combinedMask = self.service.prepCombinedMask(self.runNumber1, self.useLiteMode, timestamp, masks) + request = ReductionRequest( + runNumber=self.runNumber1, + useLiteMode=self.useLiteMode, + timestamp=timestamp, + pixelMasks=masks, + ) + + # call code and check result + with mock.patch.object( + self.service.dataFactoryService, + "getLatestApplicableCalibrationVersion", + return_value=None, + ): + combinedMask = self.service.prepCombinedMask(request) actual = arrayFromMask(combinedMask) expected = np.zeros(maskArrays[0].shape, dtype=bool) for mask in maskArrays: expected |= mask - if not np.all(expected == actual): - print( - "The expected combined mask doesn't match the calculated mask.\n" - + f" Masking values are incorrect for {np.count_nonzero(expected != actual)} pixels." - ) - assert np.all(expected == actual) + failmsg = ( + "The expected combined mask doesn't match the calculated mask.\n" + + f" Masking values are incorrect for {np.count_nonzero(expected != actual)} pixels." + ) + assert np.all(expected == actual), failmsg - def test_fetchReductionGroceries_pixelMasks(self): + def test_prepCombinedMask_load(self): + """ + Check that prepCombinedMask correctly loads all things it should + """ with ( mock.patch.object(self.service.groceryService, "fetchGroceryDict") as mockFetchGroceryDict, - mock.patch.object(self.service, "prepCombinedMask") as mockPrepCombinedMask, + mock.patch.object(self.service.dataFactoryService, "getLatestApplicableCalibrationVersion", return_value=1), ): - # timestamp must be unique: see comment at `test_prepCombinedMask`. fetchGroceryCallArgs = [] def trackFetchGroceryDict(*args, **kwargs): @@ -686,91 +736,209 @@ def trackFetchGroceryDict(*args, **kwargs): mockFetchGroceryDict.side_effect = trackFetchGroceryDict + # timestamp must be unique: see comment at `test_prepCombinedMask`. timestamp = self.service.getUniqueTimestamp() request = ReductionRequest( runNumber=self.runNumber1, - useLiteMode=False, + useLiteMode=self.useLiteMode, timestamp=timestamp, versions=Versions(1, 2), pixelMasks=[self.maskWS1, self.maskWS2, self.maskWS5], - focusGroups=[FocusGroup(name="apple", definition="path/to/grouping")], ) - self.service.dataFactoryService.getLatestApplicableCalibrationVersion = mock.Mock(return_value=1) - self.service.dataFactoryService.getLatestApplicableNormalizationVersion = mock.Mock(return_value=2) - self.service._markWorkspaceMetadata = mock.Mock() + # prepare the expected grocery dicionary groceryClerk = self.service.groceryClerk + groceryClerk.name("diffcalMaskWorkspace").diffcal_mask(request.runNumber, 1).useLiteMode( + request.useLiteMode + ).add() for mask in (self.maskWS1, self.maskWS2): runNumber, timestamp = mask.tokens("runNumber", "timestamp") groceryClerk.name(mask).reduction_pixel_mask(runNumber, timestamp).useLiteMode( request.useLiteMode ).add() - groceryClerk.name("diffcalMaskWorkspace").diffcal_mask(request.runNumber, 1).useLiteMode( - request.useLiteMode - ).add() + loadableMaskGroceryItems = groceryClerk.buildDict() residentMaskGroceryKwargs = {self.maskWS5.toString(): self.maskWS5} - combinedMaskName = wng.reductionPixelMask().runNumber(request.runNumber).build() - mockPrepCombinedMask.return_value = combinedMaskName - - groceryClerk.name("inputWorkspace").neutron(request.runNumber).useLiteMode(request.useLiteMode).add() - groceryClerk.name("diffcalWorkspace").diffcal_table( - request.runNumber, request.versions.calibration - ).useLiteMode(request.useLiteMode).add() - groceryClerk.name("normalizationWorkspace").normalization( - request.runNumber, request.versions.normalization - ).useLiteMode(request.useLiteMode).add() - loadableOtherGroceryItems = groceryClerk.buildDict() - residentOtherGroceryKwargs = {"combinedPixelMask": combinedMaskName} - self.service.fetchReductionGroceries(request) + self.service.prepCombinedMask(request) realArgs = fetchGroceryCallArgs[0][0][0] realKwargs = fetchGroceryCallArgs[0][1] assert realArgs == loadableMaskGroceryItems assert realKwargs == residentMaskGroceryKwargs - mockFetchGroceryDict.assert_any_call(loadableMaskGroceryItems, **residentMaskGroceryKwargs) - mockFetchGroceryDict.assert_any_call(groceryDict=loadableOtherGroceryItems, **residentOtherGroceryKwargs) + mockFetchGroceryDict.assert_called_with(loadableMaskGroceryItems, **residentMaskGroceryKwargs) + + def test_prepCombinedMask_only_diffcal(self): + """ + Check that prepCombinedMask will still work if only a diffcal file is present + Logic: + - the grocery service is mocked to create a MaskWorkspace based on the item + - if only the diffcal mask is loaded, it will compare equal to itself at the end + - if some other mask is loaded, either an error will occur, or it will be unequal + """ + + def mock_compatible_mask(wsname, runNumber, useLiteMode): # noqa ARG001 + return maskFromArray([0, 0, 0, 0, 0], wsname) + + def mock_fetch_grocery_list(groceryList): + import hashlib + import json + + groceries = [] + for item in groceryList: + runNumber, version, useLiteMode = item.runNumber, item.version, item.useLiteMode + workspaceName = f"{runNumber}_{useLiteMode}_v{version}" + hasher = hashlib.shake_256() + hasher.update(json.dumps(item.__dict__).encode("utf-8")) + x = int.from_bytes(hasher.digest(1), "big") + mask = [int(x) for x in list("{0:0b}".format(x))] + workspaceName = maskFromArray(mask, workspaceName) + groceries.append(workspaceName) + return groceries - def test_fetchReductionGroceries_pixelMasks_not_a_mask(self): with ( - mock.patch.object(self.service.groceryService, "fetchGroceryDict"), - mock.patch.object(self.service, "prepCombinedMask") as mockPrepCombinedMask, + mock.patch.object( + self.service.dataFactoryService, + "getLatestApplicableCalibrationVersion", + return_value=1, + ), + mock.patch.object( + self.service.groceryService, + "fetchGroceryList", + mock_fetch_grocery_list, + ), + mock.patch.object( + self.service.groceryService, + "fetchCompatiblePixelMask", + mock_compatible_mask, + ), ): - not_a_mask = ( - wng.reductionOutput() - .unit(wng.Units.DSP) - .group("bank") - .runNumber(self.runNumber1) - .timestamp(self.service.getUniqueTimestamp()) - .build() - ) - # timestamp must be unique: see comment at `test_prepCombinedMask`. timestamp = self.service.getUniqueTimestamp() request = ReductionRequest( runNumber=self.runNumber1, - useLiteMode=False, + useLiteMode=self.useLiteMode, timestamp=timestamp, versions=Versions(1, 2), - pixelMasks=[self.maskWS1, self.maskWS2, self.maskWS5, not_a_mask], - focusGroups=[FocusGroup(name="apple", definition="path/to/grouping")], + pixelMasks=[], ) - self.service.dataFactoryService.getLatestApplicableCalibrationVersion = mock.Mock(return_value=1) - self.service.dataFactoryService.getLatestApplicableNormalizationVersion = mock.Mock(return_value=2) - combinedMaskName = wng.reductionPixelMask().runNumber(request.runNumber).build() - mockPrepCombinedMask.return_value = combinedMaskName - with pytest.raises(RuntimeError, match=r".*unexpected workspace-type.*"): - self.service.fetchReductionGroceries(request) + # prepare the expected grocery dicionary + groceryClerk = self.service.groceryClerk + groceryClerk.name("diffcalMaskWorkspace").diffcal_mask(request.runNumber, 1).useLiteMode( + request.useLiteMode + ).add() + exp = self.service.groceryService.fetchGroceryDict(groceryClerk.buildDict()) + + res = self.service.prepCombinedMask(request) + + wksp_almost_equal(exp["diffcalMaskWorkspace"], res, atol=0.0) + + def test_fetchReductionGroceries_load(self): + """ + Check that fetchReductionGroceries constructs the correct grocery dictionary + NOTE this probably belongs more properly to the other test class. + However, it was already here, for simplicity of review I am not moving it. + """ + + # timestamp must be unique: see comment at `test_prepCombinedMask`. + timestamp = self.service.getUniqueTimestamp() + request = ReductionRequest( + runNumber=self.runNumber1, + useLiteMode=False, + timestamp=timestamp, + versions=Versions(1, 2), + pixelMasks=[self.maskWS1, self.maskWS2, self.maskWS5], + ) + + # prepare mocks + self.service._markWorkspaceMetadata = mock.Mock() + fetchGroceryCallArgs = [] + + def trackFetchGroceryDict(*args, **kwargs): + fetchGroceryCallArgs.append((args, kwargs)) + return mock.MagicMock() + + combinedMaskName = wng.reductionPixelMask().runNumber(request.runNumber).build() + + # construct the expected grocery dictionaries + groceryClerk = self.service.groceryClerk + groceryClerk.name("inputWorkspace").neutron(request.runNumber).useLiteMode(request.useLiteMode).add() + groceryClerk.name("diffcalWorkspace").diffcal_table( + request.runNumber, request.versions.calibration + ).useLiteMode(request.useLiteMode).add() + groceryClerk.name("normalizationWorkspace").normalization( + request.runNumber, request.versions.normalization + ).useLiteMode(request.useLiteMode).add() + loadableOtherGroceryItems = groceryClerk.buildDict() + residentOtherGroceryKwargs = {"combinedPixelMask": combinedMaskName} + + with ( + mock.patch.object(self.service.groceryService, "fetchGroceryDict", side_effect=trackFetchGroceryDict), + mock.patch.object(self.service, "prepCombinedMask", return_value=combinedMaskName), + mock.patch.object( + self.service.dataFactoryService, + "getLatestApplicableCalibrationVersion", + return_value=1, + ), + mock.patch.object( + self.service.dataFactoryService, + "getLatestApplicableNormalizationVersion", + return_value=2, + ), + mock.patch.object(self.service.groceryService, "checkPixelMask") as mockCheckPixelMask, + ): + # check -- with valid combinedPixelMask, it is used as keyword arg to fetchGroceryDict + mockCheckPixelMask.return_value = True + self.service.fetchReductionGroceries(request) + self.service.groceryService.fetchGroceryDict.assert_called_with( + loadableOtherGroceryItems, **residentOtherGroceryKwargs + ) + + # check -- with invalid combinedPixelMask, no mask is added + mockCheckPixelMask.return_value = False + self.service.fetchReductionGroceries(request) + self.service.groceryService.fetchGroceryDict.assert_called_with( + loadableOtherGroceryItems, + ) + + def test_prepCombinedMask_not_a_mask(self): + not_a_mask = ( + wng.reductionOutput() + .unit(wng.Units.DSP) + .group("bank") + .runNumber(self.runNumber1) + .timestamp(self.service.getUniqueTimestamp()) + .build() + ) + + # timestamp must be unique: see comment at `test_prepCombinedMask`. + timestamp = self.service.getUniqueTimestamp() + request = ReductionRequest( + runNumber=self.runNumber1, + useLiteMode=self.useLiteMode, + timestamp=timestamp, + pixelMasks=[not_a_mask], + ) + + with ( + mock.patch.object( + self.service.dataFactoryService, + "getLatestApplicableCalibrationVersion", + return_value=None, + ), + pytest.raises(RuntimeError, match=r".*unexpected workspace-type.*"), + ): + self.service.prepCombinedMask(request) def test_getCompatibleMasks(self): + timestamp = self.service.getUniqueTimestamp() request = ReductionRequest.model_construct( runNumber=self.runNumber1, - useLiteMode=False, + useLiteMode=self.useLiteMode, + timestamp=timestamp, versions=Versions(1, 2), pixelMasks=[self.maskWS1, self.maskWS2, self.maskWS5], - focusGroups=[FocusGroup(name="apple", definition="path/to/grouping")], ) with mock.patch.object( self.service.dataFactoryService, "getCompatibleReductionMasks" diff --git a/tests/util/InstaEats.py b/tests/util/InstaEats.py index 6a224a9bd..718655795 100644 --- a/tests/util/InstaEats.py +++ b/tests/util/InstaEats.py @@ -3,7 +3,14 @@ from typing import Any, Dict, List from unittest import mock -from mantid.simpleapi import LoadDetectorsGroupingFile, LoadEmptyInstrument, mtd +from mantid.simpleapi import ( + CreateEmptyTableWorkspace, + CreateSampleWorkspace, + ExtractMask, + LoadDetectorsGroupingFile, + LoadEmptyInstrument, + mtd, +) from pydantic import validate_call from util.WhateversInTheFridge import WhateversInTheFridge @@ -192,6 +199,29 @@ def fetchGroupingDefinition(self, item: GroceryListItem) -> Dict[str, Any]: return data + def fetchCompatiblePixelMask(self, maskWSName, runNumber, useLiteMode): + CreateSampleWorkspace( + OutputWorkspace=maskWSName, + NumBanks=1, + BankPixelWidth=1, + ) + ExtractMask( + InputWorkspace=maskWSName, + OutputWorkspace=maskWSName, + ) + + def fetchCalibrationWorkspaces(self, item): + runNumber, version, useLiteMode = item.runNumber, item.version, item.useLiteMode + tableWorkspaceName = self.lookupDiffcalTableWorkspaceName(runNumber, useLiteMode, version) + maskWorkspaceName = self._createDiffcalMaskWorkspaceName(runNumber, useLiteMode, version) + self.fetchCompatiblePixelMask(maskWorkspaceName, runNumber, useLiteMode) + CreateEmptyTableWorkspace(OutputWorkspace=tableWorkspaceName) + return { + "result": True, + "loader": "LoadCalibrationWorkspaces", + "workspace": tableWorkspaceName, + } + def fetchGroceryList(self, groceryList: List[GroceryListItem]) -> List[WorkspaceName]: """ :param groceryList: a list of GroceryListItems indicating the workspaces to create @@ -240,6 +270,8 @@ def fetchGroceryList(self, groceryList: List[GroceryListItem]) -> List[Workspace res["workspace"] = maskWorkspaceName case "normalization": res = self.fetchNormalizationWorkspace(item) + case "reduction_pixel_mask": + res = self.fetchReductionPixelMask(item) case _: raise RuntimeError(f"unrecognized 'workspaceType': '{item.workspaceType}'") # check that the fetch operation succeeded and if so append the workspace diff --git a/tests/util/helpers.py b/tests/util/helpers.py index 0a4bdba57..826e9b501 100644 --- a/tests/util/helpers.py +++ b/tests/util/helpers.py @@ -5,19 +5,96 @@ import unittest from collections.abc import Sequence -from typing import Any, Tuple +from typing import Any, List, Tuple import numpy import numpy as np -from mantid.api import ITableWorkspace, MatrixWorkspace +from mantid.api import IEventWorkspace, ITableWorkspace, MatrixWorkspace from mantid.dataobjects import GroupingWorkspace, MaskWorkspace from mantid.simpleapi import ( CreateEmptyTableWorkspace, CreateWorkspace, DeleteWorkspace, ExtractMask, + LoadInstrument, mtd, ) +from util.instrument_helpers import addInstrumentLogs + +from snapred.backend.dao.state.DetectorState import DetectorState + + +def createNPixelInstrumentXML(numberOfPixels): + """ + Given a number of pixels, create an instrument XML file with that many pixels. + These pixels have no locations and cannot be used for any valid geometry checks. + However, they will allow for certain algorithms, such as MaskDetectors to work. + """ + instrumentXML = f""" + + \n + + + + """ + for n in range(numberOfPixels): + instrumentXML += f""" + \n""" + for n in range(numberOfPixels): + instrumentXML += f"""\n""" + instrumentXML += """ + + + + + + + + + + """ + instrumentXML += "\n" + return instrumentXML + + +def createNPixelWorkspace(wsname, numberOfPixels, detectorState: DetectorState = None): + """ + Given a number of pixels, create an workspace with that many pixels. + The instrument on the workspace will have that many pixels defined, + but they have no locations or geometry. + """ + CreateWorkspace( + OutputWorkspace=wsname, + DataX=[0] * numberOfPixels, + DataY=[0] * numberOfPixels, + NSpec=numberOfPixels, + ) + LoadInstrument( + Workspace=wsname, + InstrumentName=f"tmp{numberOfPixels}", + InstrumentXML=createNPixelInstrumentXML(numberOfPixels), + RewriteSpectraMap=True, + ) + if detectorState is None: + # if no detector state given, use a default with arbitrary values + detectorState = DetectorState( + arc=(1, 2), + lin=(3, 4), + wav=10.0, + freq=60.0, + guideStat=1, + ) + logs = detectorState.getLogValues() + lognames = list(logs.keys()) + logvalues = list(logs.values()) + logtypes = ["Number Series"] * len(lognames) + addInstrumentLogs(wsname, logNames=lognames, logTypes=logtypes, logValues=logvalues) + return mtd[wsname] def createCompatibleDiffCalTable(tableWSName: str, templateWSName: str) -> ITableWorkspace: @@ -92,6 +169,28 @@ def arrayFromMask(maskWSName: str) -> numpy.ndarray: return flags +def maskFromArray(mask: List[bool], maskWSname: str, parentWSname=None, detectorState=None): + """ + Create a mask workspace with given name, with the indicated pixels masked. + If a parent workspace is passed, create the mask workspace compatible with + the parent's instrument. + If not, but a detector state is passed, will create a mask and load parameters + corresponding to the detector state. + """ + + nspec = len(mask) + if parentWSname is None: + parentWSname = mtd.unique_name() + createNPixelWorkspace(parentWSname, nspec, detectorState) + maskWS = createCompatibleMask(maskWSname, parentWSname) + assert len(mask) == maskWS.getNumberHistograms(), "The mask array was incompatible with the parent workspace." + + wkspIndexToMask = np.argwhere(mask == 1) + parentWS = mtd[parentWSname] + maskSpectra(maskWS, parentWS, wkspIndexToMask) + return maskWSname + + def initializeRandomMask(maskWSName: str, fraction: float) -> MaskWorkspace: """ Initialize an existing mask workspace by masking a random fraction of its values: @@ -114,7 +213,7 @@ def initializeRandomMask(maskWSName: str, fraction: float) -> MaskWorkspace: def setSpectraToZero(inputWS: MatrixWorkspace, nss: Sequence[int]): # Zero out all spectra in the list of spectra - if "EventWorkspace" not in inputWS.id(): + if not isinstance(inputWS, IEventWorkspace): for ns in nss: # allow "ragged" case zs = np.zeros_like(inputWS.readY(ns)) @@ -137,7 +236,7 @@ def setGroupSpectraToZero(ws: MatrixWorkspace, groupingWS: GroupingWorkspace, gi detInfo = ws.detectorInfo() for gid in gids: dets = groupingWS.getDetectorIDsOfGroup(gid) - if "EventWorkspace" not in ws.id(): + if not isinstance(ws, IEventWorkspace): for det in dets: ns = detInfo.indexOf(int(det)) # allow "ragged" case @@ -185,7 +284,6 @@ def mutableWorkspaceClones( Clone workspaces so that they can be modified by simultaneously running tests. Each cloned workspace will have a name: + """ - from mantid.simpleapi import mtd wss = [] ws_names = []