diff --git a/src/snapred/backend/dao/ingredients/ReductionIngredients.py b/src/snapred/backend/dao/ingredients/ReductionIngredients.py index 740b0a0fa..b27c58959 100644 --- a/src/snapred/backend/dao/ingredients/ReductionIngredients.py +++ b/src/snapred/backend/dao/ingredients/ReductionIngredients.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import Dict, List, Optional from pydantic import BaseModel, ConfigDict @@ -22,14 +22,12 @@ class ReductionIngredients(BaseModel): useLiteMode: bool timestamp: float - pixelGroups: List[PixelGroup] + # Changed to dict keyed by group ID + pixelGroups: Dict[int, PixelGroup] unmaskedPixelGroups: List[PixelGroup] - # these should come from calibration / normalization records - # But will not exist if we proceed without calibration / normalization - # NOTE: These are peaks for normalization, and thus should use the - # Calibrant Sample for the Normalization - detectorPeaksMany: Optional[List[List[GroupPeakList]]] = None + # Peaks now stored in a dict keyed by group ID + detectorPeaksMany: Optional[Dict[int, List[GroupPeakList]]] = None smoothingParameter: Optional[float] calibrantSamplePath: Optional[str] peakIntensityThreshold: Optional[float] @@ -45,29 +43,30 @@ def preprocess(self) -> PreprocessReductionIngredients: # At present, `PreprocessReductionIngredients` has no required parameters. return PreprocessReductionIngredients() - def getDetectorPeaks(self, groupingIndex: int) -> List[GroupPeakList]: + # Consider renaming `groupingIndex` to `groupID` for clarity. + def getDetectorPeaks(self, groupID: int) -> Optional[List[GroupPeakList]]: if self.detectorPeaksMany is None: return None - return self.detectorPeaksMany[groupingIndex] + return self.detectorPeaksMany.get(groupID) - def groupProcessing(self, groupingIndex: int) -> ReductionGroupProcessingIngredients: - return ReductionGroupProcessingIngredients(pixelGroup=self.pixelGroups[groupingIndex]) + def groupProcessing(self, groupID: int) -> ReductionGroupProcessingIngredients: + return ReductionGroupProcessingIngredients(pixelGroup=self.pixelGroups[groupID]) - def generateFocussedVanadium(self, groupingIndex: int) -> GenerateFocussedVanadiumIngredients: + def generateFocussedVanadium(self, groupID: int) -> GenerateFocussedVanadiumIngredients: return GenerateFocussedVanadiumIngredients( smoothingParameter=self.smoothingParameter, - pixelGroup=self.pixelGroups[groupingIndex], - detectorPeaks=self.getDetectorPeaks(groupingIndex), + pixelGroup=self.pixelGroups[groupID], + detectorPeaks=self.getDetectorPeaks(groupID), artificialNormalizationIngredients=self.artificialNormalizationIngredients, ) - def applyNormalization(self, groupingIndex: int) -> ApplyNormalizationIngredients: + def applyNormalization(self, groupID: int) -> ApplyNormalizationIngredients: return ApplyNormalizationIngredients( - pixelGroup=self.pixelGroups[groupingIndex], + pixelGroup=self.pixelGroups[groupID], ) - def effectiveInstrument(self, groupingIndex: int) -> EffectiveInstrumentIngredients: - return EffectiveInstrumentIngredients(unmaskedPixelGroup=self.unmaskedPixelGroups[groupingIndex]) + def effectiveInstrument(self, groupID: int) -> EffectiveInstrumentIngredients: + return EffectiveInstrumentIngredients(unmaskedPixelGroup=self.unmaskedPixelGroups[groupID]) model_config = ConfigDict( extra="forbid", diff --git a/src/snapred/backend/recipe/ReductionRecipe.py b/src/snapred/backend/recipe/ReductionRecipe.py index 65da198a2..d302860e3 100644 --- a/src/snapred/backend/recipe/ReductionRecipe.py +++ b/src/snapred/backend/recipe/ReductionRecipe.py @@ -63,8 +63,10 @@ def unbagGroceries(self, groceries: Dict[str, Any]): self.groceries = groceries.copy() self.sampleWs = groceries["inputWorkspace"] self.normalizationWs = groceries.get("normalizationWorkspace", "") - self.maskWs = groceries.get("combinedMask", "") - self.groupingWorkspaces = groceries["groupingWorkspaces"] + self.maskWs = groceries.get("combinedPixelMask", "") + self.groupingWorkspaces = { + groupIndex: workspaceName for groupIndex, workspaceName in enumerate(groceries["groupingWorkspaces"]) + } def _cloneWorkspace(self, inputWorkspace: str, outputWorkspace: str) -> str: self.mantidSnapper.CloneWorkspace( @@ -81,7 +83,7 @@ def _cloneIntermediateWorkspace(self, inputWorkspace: str, outputWorkspace: str) "Cloning workspace...", InputWorkspace=inputWorkspace, OutputWorkspace=outputWorkspace ) self.mantidSnapper.executeQueue() - return inputWorkspace + return outputWorkspace def _deleteWorkspace(self, workspace: str): self.mantidSnapper.DeleteWorkspace( @@ -155,12 +157,16 @@ def _applyRecipe(self, recipe: Type[Recipe], ingredients_, **kwargs): recipe().cook(ingredients_, self.groceries) else: raise RuntimeError( - ( - f"{recipe.__name__} ::" - " Missing non-default input workspace with groceries:" - f" {self.groceries} and kwargs: {kwargs}" - ) + f"{recipe.__name__} :: InputWorkspace '{inputWorkspace}' does not exist in Mantid workspace dictionary." # noqa: E501 ) + if "groupingWorkspace" in kwargs: + groupingWorkspace = kwargs["groupingWorkspace"] + if not self.mantidSnapper.mtd.doesExist(groupingWorkspace): + raise RuntimeError( + f"{recipe.__name__} :: GroupingWorkspace '{groupingWorkspace}' does not exist in Mantid workspace dictionary." # noqa: E501 + ) + self.groceries.update(kwargs) + recipe().cook(ingredients_, self.groceries) def _getNormalizationWorkspaceName(self, groupingIndex: int): return f"reduced_normalization_{groupingIndex}_{wnvf.formatTimestamp(self.ingredients.timestamp)}" @@ -174,8 +180,10 @@ def _prepGroupingWorkspaces(self, groupingIndex: int): groupingName = self.ingredients.pixelGroups[groupingIndex].focusGroup.name.lower() reducedOutputWs = wng.reductionOutput().runNumber(runNumber).group(groupingName).timestamp(timestamp).build() + sampleClone = self._cloneWorkspace(self.sampleWs, reducedOutputWs) self.groceries["inputWorkspace"] = sampleClone + normalizationClone = None if self.normalizationWs: normalizationClone = self._cloneWorkspace( @@ -183,22 +191,52 @@ def _prepGroupingWorkspaces(self, groupingIndex: int): self._getNormalizationWorkspaceName(groupingIndex), ) self.groceries["normalizationWorkspace"] = normalizationClone - return sampleClone, normalizationClone - - def _isGroupFullyMasked(self, groupingWorkspace: str) -> bool: - maskWorkspace = self.mantidSnapper.mtd[self.maskWs] - groupWorkspace = self.mantidSnapper.mtd[groupingWorkspace] - totalMaskedPixels = 0 - totalGroupPixels = 0 + return sampleClone, normalizationClone - for i in range(groupWorkspace.getNumberHistograms()): - group_spectra = groupWorkspace.readY(i) - for spectrumIndex in group_spectra: - if maskWorkspace.readY(int(spectrumIndex))[0] == 1: - totalMaskedPixels += 1 - totalGroupPixels += 1 - return totalMaskedPixels == totalGroupPixels + def _checkMaskedPixels(self, groupingWorkspace: str) -> bool: + try: + # Extract the focus group name from the grouping workspace name + focusGroupName = groupingWorkspace.split("__")[1].rsplit("_", 1)[0] + except IndexError: + self.logger().error(f"Unexpected groupingWorkspace format: '{groupingWorkspace}'. Skipping this workspace.") + return True # Skip execution for invalid format + + # Retrieve the PixelGroup matching the focus group name + pixelGroup = next( + ( + group + for group in self.ingredients.pixelGroups + if group.focusGroup.name.lower() == focusGroupName.lower() + ), + None, + ) + if not pixelGroup: + self.logger().error(f"No matching PixelGroup found for {groupingWorkspace}") + return True + + # Check if all pixels in the group are masked + allMasked = all(param.isMasked for param in pixelGroup.pixelGroupingParameters.values()) + if allMasked: + self.logger().warning( + f"All pixels in group '{focusGroupName}' (workspace: '{groupingWorkspace}') are masked. " + "Skipping this group for the reduction.\n" + ) + return True + return False + + def _removeFullyMaskedGroups(self): + keysRemoved = [ + groupIndex + for groupIndex, groupingWs in self.groupingWorkspaces.items() + if self.maskWs and self._checkMaskedPixels(groupingWs) + ] + + for key in keysRemoved: + self.groupingWorkspaces.pop(key) + self.ingredients.pixelGroups.pop(key) + if self.ingredients.detectorPeaksMany is not None: + self.ingredients.detectorPeaksMany.pop(key) def queueAlgos(self): pass @@ -207,8 +245,8 @@ def execute(self): data: Dict[str, Any] = {"result": False} # Retain unfocused data for comparison. - if self.keepUnfocused: - data["unfocusedWS"] = self._prepareUnfocusedData(self.sampleWs, self.maskWs, self.convertUnitsTo) + if self.maskWs: + self._removeFullyMaskedGroups() # 1. PreprocessReductionRecipe outputs = [] @@ -227,16 +265,11 @@ def execute(self): ) self._cloneIntermediateWorkspace(self.normalizationWs, "normalization_preprocessed") - for groupingIndex, groupingWs in enumerate(self.groupingWorkspaces): + for groupingIndex, groupingWs in self.groupingWorkspaces.items(): self.groceries["groupingWorkspace"] = groupingWs - if self.maskWs and self._isGroupFullyMasked(groupingWs): + if self.maskWs and self._checkMaskedPixels(groupingWs): # Notify the user of a fully masked group, but continue with the workflow - self.logger().warning( - f"\nAll pixels masked within {groupingWs} schema.\n" - + "Skipping all algorithm execution for this group.\n" - + "This will affect future reductions." - ) continue sampleClone, normalizationClone = self._prepGroupingWorkspaces(groupingIndex) diff --git a/src/snapred/backend/service/ReductionService.py b/src/snapred/backend/service/ReductionService.py index 142e9b172..430d9ebe4 100644 --- a/src/snapred/backend/service/ReductionService.py +++ b/src/snapred/backend/service/ReductionService.py @@ -215,7 +215,8 @@ def _createReductionRecord( calibration=calibration, normalization=normalization, pixelGroupingParameters={ - pg.focusGroup.name: [pg[gid] for gid in pg.groupIDs] for pg in ingredients.pixelGroups + pg.focusGroup.name: [pg.pixelGroupingParameters[gid] for gid in pg.groupIDs] + for pg in ingredients.pixelGroups.values() }, workspaceNames=workspaceNames, ) diff --git a/src/snapred/backend/service/SousChef.py b/src/snapred/backend/service/SousChef.py index fbb6aabec..ea6177523 100644 --- a/src/snapred/backend/service/SousChef.py +++ b/src/snapred/backend/service/SousChef.py @@ -267,21 +267,36 @@ def prepReductionIngredients( self, ingredients: FarmFreshIngredients, combinedPixelMask: Optional[WorkspaceName] = None ) -> ReductionIngredients: ingredients_ = ingredients.model_copy() - # some of the reduction ingredients MUST match those used in the calibration/normalization processes + + # Some of the reduction ingredients MUST match those used in the calibration/normalization processes ingredients_ = self._pullCalibrationRecordFFI(ingredients_) ingredients_, smoothingParameter, calibrantSamplePath = self._pullNormalizationRecordFFI(ingredients_) ingredients_.calibrantSamplePath = calibrantSamplePath + # Convert pixelGroups to a dictionary keyed by group ID + pixelGroups_dict = { + i: group for i, group in enumerate(self.prepManyPixelGroups(ingredients_, combinedPixelMask)) + } + unmaskedPixelGroups_dict = {i: group for i, group in enumerate(self.prepManyPixelGroups(ingredients_))} + + # Convert detectorPeaksMany to a dictionary if it exists + detectorPeaksMany_dict = ( + {i: peaks for i, peaks in enumerate(self.prepManyDetectorPeaks(ingredients_))} + if self.prepManyDetectorPeaks(ingredients_) + else None + ) + + # Create the ReductionIngredients object return ReductionIngredients( runNumber=ingredients_.runNumber, useLiteMode=ingredients_.useLiteMode, timestamp=ingredients_.timestamp, - pixelGroups=self.prepManyPixelGroups(ingredients_, combinedPixelMask), - unmaskedPixelGroups=self.prepManyPixelGroups(ingredients_), + pixelGroups=pixelGroups_dict, + unmaskedPixelGroups=list(unmaskedPixelGroups_dict.values()), smoothingParameter=smoothingParameter, calibrantSamplePath=ingredients_.calibrantSamplePath, peakIntensityThreshold=self._getThresholdFromCalibrantSample(ingredients_.calibrantSamplePath), - detectorPeaksMany=self.prepManyDetectorPeaks(ingredients_), + detectorPeaksMany=detectorPeaksMany_dict, keepUnfocused=ingredients_.keepUnfocused, convertUnitsTo=ingredients_.convertUnitsTo, ) diff --git a/tests/data/snapred-data b/tests/data/snapred-data index 59443a567..8191671d5 160000 --- a/tests/data/snapred-data +++ b/tests/data/snapred-data @@ -1 +1 @@ -Subproject commit 59443a567f5b78447a9ee13ef5483af28fcdcaeb +Subproject commit 8191671d5151e474740d4edec939d0eb4de08615