Merge branch 'next' into ewm7922-defect-pixel-masks

.github/workflows/actions.yml

@@ -100,7 +100,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python-version: ["3.8"]
+        python-version: ["3.10"]
     defaults:
       run:
         shell: bash -l {0}
@@ -125,7 +125,7 @@ jobs:
         cd conda.recipe
         echo "versioningit $(versioningit ../)"
         # build the package
-        VERSION=$(versioningit ../) conda mambabuild --channel conda-forge --output-folder . .
+        VERSION=$(versioningit ../) conda mambabuild --channel conda-forge --channel mantid/label/nightly --output-folder . .
         conda verify noarch/snapred*.tar.bz2
     - name: Deploy to Anaconda
       shell: bash -l {0}

.gitmodules

@@ -1,3 +1,4 @@
 [submodule "tests/data/snapred-data"]
 	path = tests/data/snapred-data
 	url = https://code.ornl.gov/sns-hfir-scse/infrastructure/test-data/snapred-data.git
+	branch = main

.pre-commit-config.yaml

@@ -16,7 +16,7 @@ repos:
     - id: trailing-whitespace
       exclude: "tests/cis_tests/.*"
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.8.1
+  rev: v0.8.3
   hooks:
     - id: ruff
       args: [--fix, --exit-non-zero-on-fix]

pyproject.toml

@@ -4,7 +4,7 @@ description = "A desktop application for Lifecycle Managment of data collected f
 dynamic = ["version"]
 requires-python = ">=3.10"
 dependencies = [
-    "mantidworkbench >= 6.10.0.2rc1",
+    "mantidworkbench >= 6.11.20241203",
     "pyoncat ~= 1.6"
 ]
 readme = "README.md"
@@ -55,7 +55,8 @@ markers = [
   "integration: mark a test as an integration test",
   "mount_snap: mark a test as using /SNS/SNAP/ data mount",
   "golden_data(*, path=None, short_name=None, date=None): mark golden data to use with a test",
-  "datarepo: mark a test as using snapred-data repo"
+  "datarepo: mark a test as using snapred-data repo",
+  "ui: mark a test as a UI test",
 ]
 # The following will be overridden by the commandline option "-m integration"
 addopts = "-m 'not (integration or datarepo)'"

src/snapred/backend/dao/indexing/Versioning.py

@@ -1,79 +1,51 @@
-from typing import Any, Optional
-
-from numpy import integer
-from pydantic import BaseModel, computed_field, field_serializer
+from pydantic import BaseModel, ConfigDict, field_validator
 
 from snapred.meta.Config import Config
+from snapred.meta.Enum import StrEnum
 
 VERSION_START = Config["version.start"]
 VERSION_NONE_NAME = Config["version.friendlyName.error"]
-VERSION_DEFAULT_NAME = Config["version.friendlyName.default"]
 
-# VERSION_DEFAULT is a SNAPRed-internal "magic" integer:
-# * it is implicitely set during `Config` initialization.
-VERSION_DEFAULT = Config["version.default"]
+
+class VersionState(StrEnum):
+    DEFAULT = Config["version.friendlyName.default"]
+    LATEST = "latest"
+    NEXT = "next"
+
+
+# I'm not sure why ci is failing without this, it doesn't seem to be used anywhere
+VERSION_DEFAULT = VersionState.DEFAULT
+
+Version = int | VersionState
 
 
 class VersionedObject(BaseModel):
     # Base class for all versioned DAO
 
-    # In pydantic, a leading double underscore activates
-    # the `__pydantic_private__` feature, which limits the visibility
-    # of the attribute to the interior scope of its own class.
-    __version: Optional[int] = None
-
-    @classmethod
-    def parseVersion(cls, version, *, exclude_none: bool = False, exclude_default: bool = False) -> int | None:
-        v: int | None
-        # handle two special cases
-        if (not exclude_none) and (version is None or version == VERSION_NONE_NAME):
-            v = None
-        elif (not exclude_default) and (version == VERSION_DEFAULT_NAME or version == VERSION_DEFAULT):
-            v = VERSION_DEFAULT
-        # parse integers
-        elif isinstance(version, int | integer):
-            if int(version) >= VERSION_START:
-                v = int(version)
-            else:
-                raise ValueError(f"Given version {version} is smaller than start version {VERSION_START}")
-        # otherwise this is an error
-        else:
-            raise ValueError(f"Cannot initialize version as {version}")
-        return v
-
-    @classmethod
-    def writeVersion(cls, version) -> int | str:
-        v: int | str
-        if version is None:
-            v = VERSION_NONE_NAME
-        elif version == VERSION_DEFAULT:
-            v = VERSION_DEFAULT_NAME
-        elif isinstance(version, int | integer):
-            v = int(version)
-        else:
-            raise ValueError("Version is not valid")
-        return v
-
-    def __init__(self, **kwargs):
-        version = kwargs.pop("version", None)
-        super().__init__(**kwargs)
-        self.__version = self.parseVersion(version)
-
-    @field_serializer("version", check_fields=False, when_used="json")
-    def write_user_defaults(self, value: Any):  # noqa ARG002
-        return self.writeVersion(self.__version)
-
-    # NOTE some serialization still using the dict() method
-    def dict(self, **kwargs):
-        res = super().dict(**kwargs)
-        res["version"] = self.writeVersion(res["version"])
-        return res
-
-    @computed_field
-    @property
-    def version(self) -> int:
-        return self.__version
-
-    @version.setter
-    def version(self, v):
-        self.__version = self.parseVersion(v, exclude_none=True)
+    version: Version
+
+    @field_validator("version", mode="before")
+    def validate_version(cls, value: Version) -> Version:
+        if value in VersionState.values():
+            return value
+
+        if isinstance(value, str):
+            raise ValueError(f"Version must be an int or {VersionState.values()}")
+
+        if value is None:
+            raise ValueError("Version must be specified")
+
+        if value < VERSION_START:
+            raise ValueError(f"Version must be greater than {VERSION_START}")
+
+        return value
+
+    # NOTE: This approach was taken because 'field_serializer' was checking against the
+    #       INITIAL value of version for some reason.  This is a workaround.
+    #
+    def model_dump_json(self, *args, **kwargs):  # noqa ARG002
+        if self.version in VersionState.values():
+            raise ValueError(f"Version {self.version} must be flattened to an int before writing to JSON")
+        return super().model_dump_json(*args, **kwargs)
+
+    model_config = ConfigDict(use_enum_values=True, validate_assignment=True)

src/snapred/backend/dao/ingredients/GenerateFocussedVanadiumIngredients.py

@@ -3,6 +3,7 @@
 from pydantic import BaseModel
 
 from snapred.backend.dao.GroupPeakList import GroupPeakList
+from snapred.backend.dao.ingredients.ArtificialNormalizationIngredients import ArtificialNormalizationIngredients
 from snapred.backend.dao.state.PixelGroup import PixelGroup
 from snapred.meta.Config import Config
 
@@ -14,3 +15,4 @@ class GenerateFocussedVanadiumIngredients(BaseModel):
     pixelGroup: PixelGroup
     # This can be None if we lack a calibration
     detectorPeaks: Optional[list[GroupPeakList]] = None
+    artificialNormalizationIngredients: Optional[ArtificialNormalizationIngredients] = None

src/snapred/backend/dao/ingredients/RebinFocussedGroupDataIngredients.py

@@ -0,0 +1,8 @@
+from pydantic import BaseModel
+
+from snapred.backend.dao.state.PixelGroup import PixelGroup
+
+
+class RebinFocussedGroupDataIngredients(BaseModel):
+    pixelGroup: PixelGroup
+    preserveEvents: bool = False

src/snapred/backend/dao/ingredients/ReductionIngredients.py

@@ -57,6 +57,7 @@ def generateFocussedVanadium(self, groupID: int) -> GenerateFocussedVanadiumIngr
             smoothingParameter=self.smoothingParameter,
             pixelGroup=self.pixelGroups[groupID],
             detectorPeaks=self.getDetectorPeaks(groupID),
+            artificialNormalizationIngredients=self.artificialNormalizationIngredients,
         )
 
     def applyNormalization(self, groupID: int) -> ApplyNormalizationIngredients:

src/snapred/backend/dao/normalization/NormalizationRecord.py

@@ -1,9 +1,9 @@
 from typing import Any, List
 
-from pydantic import field_serializer, field_validator
+from pydantic import field_validator
 
 from snapred.backend.dao.indexing.Record import Record
-from snapred.backend.dao.indexing.Versioning import VERSION_DEFAULT, VersionedObject
+from snapred.backend.dao.indexing.Versioning import VERSION_START, Version, VersionedObject
 from snapred.backend.dao.Limit import Limit
 from snapred.backend.dao.normalization.Normalization import Normalization
 from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceName
@@ -31,7 +31,7 @@ class NormalizationRecord(Record, extra="ignore"):
     smoothingParameter: float
     # detectorPeaks: List[DetectorPeak] # TODO: need to save this for reference during reduction
     workspaceNames: List[WorkspaceName] = []
-    calibrationVersionUsed: int = VERSION_DEFAULT
+    calibrationVersionUsed: Version = VERSION_START
     crystalDBounds: Limit[float]
     normalizationCalibrantSamplePath: str
 
@@ -44,8 +44,4 @@ def validate_backgroundRunNumber(cls, v: Any) -> Any:
     @field_validator("calibrationVersionUsed", mode="before")
     @classmethod
     def version_is_integer(cls, v: Any) -> Any:
-        return VersionedObject.parseVersion(v)
-
-    @field_serializer("calibrationVersionUsed", when_used="json")
-    def write_user_defaults(self, value: Any):  # noqa ARG002
-        return VersionedObject.writeVersion(self.calibrationVersionUsed)
+        return VersionedObject(version=v).version

src/snapred/backend/dao/request/CreateArtificialNormalizationRequest.py

@@ -1,4 +1,4 @@
-from pydantic import BaseModel, root_validator
+from pydantic import BaseModel
 
 from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceName
 
@@ -11,13 +11,7 @@ class CreateArtificialNormalizationRequest(BaseModel):
     decreaseParameter: bool = True
     lss: bool = True
     diffractionWorkspace: WorkspaceName
-    outputWorkspace: WorkspaceName = None
-
-    @root_validator(pre=True)
-    def set_output_workspace(cls, values):
-        if values.get("diffractionWorkspace") and not values.get("outputWorkspace"):
-            values["outputWorkspace"] = WorkspaceName(f"{values['diffractionWorkspace']}_artificialNorm")
-        return values
+    outputWorkspace: WorkspaceName
 
     class Config:
         arbitrary_types_allowed = True  # Allow arbitrary types like WorkspaceName

src/snapred/backend/dao/request/CreateCalibrationRecordRequest.py

@@ -5,6 +5,7 @@
 from snapred.backend.dao.calibration.Calibration import Calibration
 from snapred.backend.dao.calibration.FocusGroupMetric import FocusGroupMetric
 from snapred.backend.dao.CrystallographicInfo import CrystallographicInfo
+from snapred.backend.dao.indexing.Versioning import Version, VersionState
 from snapred.backend.dao.state.PixelGroup import PixelGroup
 from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceName, WorkspaceType
 
@@ -18,7 +19,7 @@ class CreateCalibrationRecordRequest(BaseModel, extra="forbid"):
 
     runNumber: str
     useLiteMode: bool
-    version: Optional[int] = None
+    version: Version = VersionState.NEXT
     calculationParameters: Calibration
     crystalInfo: CrystallographicInfo
     pixelGroups: Optional[List[PixelGroup]] = None

src/snapred/backend/dao/request/CreateIndexEntryRequest.py

@@ -1,7 +1,7 @@
-from typing import Optional
-
 from pydantic import BaseModel
 
+from snapred.backend.dao.indexing.Versioning import Version, VersionState
+
 
 class CreateIndexEntryRequest(BaseModel):
     """
@@ -10,7 +10,7 @@ class CreateIndexEntryRequest(BaseModel):
 
     runNumber: str
     useLiteMode: bool
-    version: Optional[int] = None
+    version: Version = VersionState.NEXT
     comments: str
     author: str
     appliesTo: str

src/snapred/backend/dao/request/DiffractionCalibrationRequest.py

@@ -3,7 +3,7 @@
 
 from pydantic import BaseModel, field_validator
 
-from snapred.backend.dao.indexing.Versioning import VERSION_DEFAULT
+from snapred.backend.dao.indexing.Versioning import Version, VersionState
 from snapred.backend.dao.Limit import Pair
 from snapred.backend.dao.state.FocusGroup import FocusGroup
 from snapred.backend.error.ContinueWarning import ContinueWarning
@@ -40,7 +40,7 @@ class DiffractionCalibrationRequest(BaseModel, extra="forbid"):
 
     continueFlags: Optional[ContinueWarning.Type] = ContinueWarning.Type.UNSET
 
-    startingTableVersion: int = VERSION_DEFAULT
+    startingTableVersion: Version = VersionState.DEFAULT
 
     @field_validator("fwhmMultipliers", mode="before")
     @classmethod

src/snapred/backend/dao/request/FarmFreshIngredients.py

@@ -2,13 +2,14 @@
 
 from pydantic import BaseModel, ConfigDict, ValidationError, field_validator, model_validator
 
+from snapred.backend.dao.indexing.Versioning import Version, VersionState
 from snapred.backend.dao.Limit import Limit, Pair
 from snapred.backend.dao.state import FocusGroup
 from snapred.meta.Config import Config
 from snapred.meta.mantid.AllowedPeakTypes import SymmetricPeakEnum
 
 # TODO: this declaration is duplicated in `ReductionRequest`.
-Versions = NamedTuple("Versions", [("calibration", Optional[int]), ("normalization", Optional[int])])
+Versions = NamedTuple("Versions", [("calibration", Version), ("normalization", Version)])
 
 
 class FarmFreshIngredients(BaseModel):
@@ -21,19 +22,19 @@ class FarmFreshIngredients(BaseModel):
 
     runNumber: str
 
-    versions: Versions = Versions(None, None)
+    versions: Versions = Versions(VersionState.LATEST, VersionState.LATEST)
 
     # allow 'versions' to be accessed as a single version,
     #   or, to be accessed ambiguously
     @property
-    def version(self) -> Optional[int]:
+    def version(self) -> Version:
         if self.versions.calibration is not None and self.versions.normalization is not None:
             raise RuntimeError("accessing 'version' property when 'versions' is non-singular")
         return self.versions[0]
 
     @version.setter
-    def version(self, v: Optional[int]):
-        self.versions = (v, None)
+    def version(self, v: Version):
+        self.versions = Versions(v, v)
 
     useLiteMode: bool
 
@@ -83,6 +84,10 @@ def focusGroup(self, fg: FocusGroup):
     def validate_versions(cls, v) -> Versions:
         if not isinstance(v, Versions):
             v = Versions(v)
+        if v.calibration is None:
+            raise ValueError("Calibration version must be specified")
+        if v.normalization is None:
+            raise ValueError("Normalization version must be specified")
         return v
 
     @field_validator("crystalDBounds", mode="before")
@@ -119,4 +124,4 @@ def validate_focusGroups(cls, v: Any):
                 del v["focusGroup"]
         return v
 
-    model_config = ConfigDict(extra="forbid")
+    model_config = ConfigDict(extra="forbid", validate_assignment=True)

src/snapred/backend/dao/request/LoadCalibrationRecordRequest.py

@@ -0,0 +1,9 @@
+from pydantic import BaseModel
+
+from snapred.backend.dao import RunConfig
+from snapred.backend.dao.indexing.Versioning import Version
+
+
+class LoadCalibrationRecordRequest(BaseModel):
+    runConfig: RunConfig
+    version: Version