From ba538d1e38c4d4134b000e21a147e32aecd83b92 Mon Sep 17 00:00:00 2001
From: MialLewis <95620982+MialLewis@users.noreply.github.com>
Date: Wed, 26 Apr 2023 17:30:02 +0100
Subject: [PATCH 01/15] add ability to manually specify detectors

---
 .../calibrate_vesuvio_analysis.py             | 44 ++++++++++++++-----
 .../tests/system/test_system_analysis.py      | 18 ++++++++
 2 files changed, 52 insertions(+), 10 deletions(-)

diff --git a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
index de66f8a4..df11daca 100644
--- a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
+++ b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
@@ -35,15 +35,21 @@ def PyInit(self):
         self.declareProperty('Mass', sys.float_info.max,
                              doc="Mass of the sample in amu to be used when calculating energy. Default is Pb: 207.19")
 
-        greaterThanZero = FloatArrayBoundedValidator()
-        greaterThanZero.setLower(0)
-        self.declareProperty(FloatArrayProperty('DSpacings', [], greaterThanZero, Direction.Input),
+        greater_than_zero_float = FloatArrayBoundedValidator()
+        greater_than_zero_float.setLower(0)
+        self.declareProperty(FloatArrayProperty('DSpacings', [], greater_than_zero_float, Direction.Input),
                              doc="List of d-spacings used to estimate the positions of peaks in TOF.")
 
-        self.declareProperty(FloatArrayProperty('E1FixedValueAndError', [], greaterThanZero, Direction.Input),
+        self.declareProperty(FloatArrayProperty('E1FixedValueAndError', [], greater_than_zero_float, Direction.Input),
                              doc="Value at which to fix E1 and E1 error (form: E1 value, E1 Error). If no input is provided,"
                                  "values will be calculated.")
 
+        detector_validator = IntArrayBoundedValidator()
+        detector_validator.setLower(EVSGlobals.DETECTOR_RANGE[0])
+        detector_validator.setUpper(EVSGlobals.DETECTOR_RANGE[-1])
+        self.declareProperty(IntArrayProperty('InvalidDetectors', [], detector_validator, Direction.Input),
+                             doc="Detectors to be marked as invalid, to be excused from analysis calculations.")
+
         self.declareProperty('Iterations', 2, validator=IntBoundedValidator(lower=1),
                              doc="Number of iterations to perform. Default is 2.")
 
@@ -111,7 +117,6 @@ def PyExec(self):
         # repeatedly fit L1, E1 and Theta parameters
         table_group = []
         for i in range(self._iterations):
-
             # calibrate theta for all detectors
             theta_fit = self._current_workspace + '_theta'
             self._run_calibration_fit(Samples=self._samples, Background=self._background, Function=self._theta_peak_function,
@@ -247,10 +252,23 @@ def _calculate_final_flight_path(self, peak_table, spec_list):
 
         peak_centres = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzPos', spec_list)
         peak_centres_errors = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzPos_Err', spec_list)
-        invalid_spectra = EVSMiscFunctions.identify_invalid_spectra(peak_table, peak_centres, peak_centres_errors, spec_list)
-        peak_centres[invalid_spectra] = np.nan
 
-        print(f'Invalid Spectra Index Found and Marked NAN: {invalid_spectra.flatten()} from Spectra Index List:'
+        if spec_list == EVSGlobals.FRONTSCATTERING_RANGE:
+            if not self._invalid_detectors_front.any():
+                self._invalid_detectors_front = EVSMiscFunctions.identify_invalid_spectra(peak_table, peak_centres, peak_centres_errors,
+                                                                                          spec_list)
+            invalid_detectors = self._invalid_detectors_front
+        elif spec_list == EVSGlobals.BACKSCATTERING_RANGE:
+            if not self._invalid_detectors_back.any():
+                self._invalid_detectors_back = EVSMiscFunctions.identify_invalid_spectra(peak_table, peak_centres, peak_centres_errors,
+                                                                                         spec_list)
+            invalid_detectors = self._invalid_detectors_back
+        else:
+            raise AttributeError("Spec list invalid - must represent either front or back detectors.")
+
+        peak_centres[invalid_detectors] = np.nan
+
+        print(f'Invalid Spectra Index Found and Marked NAN: {invalid_detectors} from Spectra Index List:'
               f'{[ x -3 for x in spec_list]}')
 
         delta_t = (peak_centres - t0) / 1e+6
@@ -331,8 +349,10 @@ def _calculate_final_energy(self, peak_table, spec_list, calculate_global):
             peak_centres = EVSMiscFunctions.read_fitting_result_table_column(peak_table[0], 'f1.LorentzPos', spec_list)
             peak_centres_errors = EVSMiscFunctions.read_fitting_result_table_column(peak_table[0], 'f1.LorentzPos_Err', spec_list)
 
-            invalid_spectra = EVSMiscFunctions.identify_invalid_spectra(peak_table[0], peak_centres, peak_centres_errors, spec_list)
-            peak_centres[invalid_spectra] = np.nan
+            if not self._invalid_detectors_back.any():
+                self._invalid_detectors_back = EVSMiscFunctions.identify_invalid_spectra(peak_table[0], peak_centres, peak_centres_errors,
+                                                                                         spec_list)
+            peak_centres[self._invalid_detectors_back] = np.nan
 
             delta_t = (peak_centres - t0) / 1e+6
             v1 = (L0 * r_theta + L1) / delta_t
@@ -381,6 +401,10 @@ def _setup(self):
         self._sample_mass = self.getProperty("Mass").value
         self._d_spacings = self.getProperty("DSpacings").value.tolist()
         self._E1_value_and_error = self.getProperty("E1FixedValueAndError").value.tolist()
+        self._invalid_detectors_front = np.array([[x - EVSGlobals.FRONTSCATTERING_RANGE[0]] for x in self.getProperty("InvalidDetectors").value.tolist()
+                                                  if EVSGlobals.FRONTSCATTERING_RANGE[0] <= x <= EVSGlobals.FRONTSCATTERING_RANGE[-1]])
+        self._invalid_detectors_back = np.array([[x - EVSGlobals.BACKSCATTERING_RANGE[0]] for x in self.getProperty("InvalidDetectors").value.tolist()
+                                                if EVSGlobals.BACKSCATTERING_RANGE[0] <= x <= EVSGlobals.BACKSCATTERING_RANGE[-1]])
         self._shared_parameter_fit_type = self.getProperty("SharedParameterFitType").value
         self._calc_L0 = self.getProperty("CalculateL0").value
         self._make_IP_file = self.getProperty("CreateIPFile").value
diff --git a/unpackaged/vesuvio_calibration/tests/system/test_system_analysis.py b/unpackaged/vesuvio_calibration/tests/system/test_system_analysis.py
index 779568cd..3a17da55 100644
--- a/unpackaged/vesuvio_calibration/tests/system/test_system_analysis.py
+++ b/unpackaged/vesuvio_calibration/tests/system/test_system_analysis.py
@@ -175,6 +175,24 @@ def test_copper_create_output(self, load_file_mock):
                                                                                            165, 167, 168, 169, 170, 182, 191, 192]})
         self._assert_parameters_match_expected(params_table, detector_specific_r_tols)
 
+    @patch('calibration_scripts.calibrate_vesuvio_fit.EVSCalibrationFit._load_file')
+    def test_copper_create_invalid_detectors_specified(self, load_file_mock):
+        self._setup_copper_test()
+        self._output_workspace = "copper_analysis_test"
+
+        load_file_mock.side_effect = self._load_file_side_effect
+
+        self._create_evs_calibration_alg()
+        self._alg.setProperty("InvalidDetectors", [3, 5, 7, 141, 144, 149, 150, 159, 161, 163, 166, 167, 168, 170, 171, 172, 173, 185, 194,
+                                                   195])
+        params_table = self._execute_evs_calibration_analysis()
+
+        #  Specify detectors tolerances set by user, then update with those to mask as invalid.
+        detector_specific_r_tols = {"L1": {116: 0.65, 170: 0.75}}
+        detector_specific_r_tols["L1"].update({k: TestConstants.INVALID_DETECTOR for k in [0, 2, 4, 138, 141, 146, 147, 156, 158, 160, 163,
+                                                                                           164, 165, 167, 168, 169, 170, 182, 191, 192]})
+        self._assert_parameters_match_expected(params_table, detector_specific_r_tols)
+
     @patch('calibration_scripts.calibrate_vesuvio_fit.EVSCalibrationFit._load_file')
     def test_copper_with_individual_and_global_fit(self, load_file_mock):
         self._setup_copper_test()

From d6da6db8108e4bef7d85e18aac57d6c0dff83427 Mon Sep 17 00:00:00 2001
From: MialLewis <95620982+MialLewis@users.noreply.github.com>
Date: Thu, 27 Apr 2023 08:35:10 +0100
Subject: [PATCH 02/15] add system test to test manual detector specification

---
 .../tests/system/test_system_analysis.py                 | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/unpackaged/vesuvio_calibration/tests/system/test_system_analysis.py b/unpackaged/vesuvio_calibration/tests/system/test_system_analysis.py
index 3a17da55..45e8aed0 100644
--- a/unpackaged/vesuvio_calibration/tests/system/test_system_analysis.py
+++ b/unpackaged/vesuvio_calibration/tests/system/test_system_analysis.py
@@ -10,7 +10,7 @@
 from tests.testhelpers.system_test_misc_functions import assert_allclose_excluding_bad_detectors
 from calibration_scripts.calibrate_vesuvio_analysis import EVSCalibrationAnalysis
 from calibration_scripts.calibrate_vesuvio_fit import EVSCalibrationFit
-from copy import copy
+from copy import copy, deepcopy
 from os import path
 
 
@@ -189,8 +189,11 @@ def test_copper_create_invalid_detectors_specified(self, load_file_mock):
 
         #  Specify detectors tolerances set by user, then update with those to mask as invalid.
         detector_specific_r_tols = {"L1": {116: 0.65, 170: 0.75}}
-        detector_specific_r_tols["L1"].update({k: TestConstants.INVALID_DETECTOR for k in [0, 2, 4, 138, 141, 146, 147, 156, 158, 160, 163,
+        detector_specific_r_tols["L1"].update({k: TestConstants.INVALID_DETECTOR for k in [138, 141, 146, 147, 156, 158, 160, 163,
                                                                                            164, 165, 167, 168, 169, 170, 182, 191, 192]})
+        self.assertRaises(AssertionError, self._assert_parameters_match_expected, *[params_table, detector_specific_r_tols])
+
+        detector_specific_r_tols["L1"].update({k: TestConstants.INVALID_DETECTOR for k in [0, 2, 4]})
         self._assert_parameters_match_expected(params_table, detector_specific_r_tols)
 
     @patch('calibration_scripts.calibrate_vesuvio_fit.EVSCalibrationFit._load_file')
@@ -244,7 +247,7 @@ def _assert_E1_parameters_match_expected(self, params_table, rel_tolerance, fit_
             self.assertAlmostEqual(global_E1, EVSGlobals.ENERGY_ESTIMATE, delta=EVSGlobals.ENERGY_ESTIMATE*rel_tolerance)
 
     def _assert_parameters_match_expected(self, params_table, tolerances=None):
-        rel_tol_theta, rel_tol_L1 = self._extract_tolerances(tolerances)
+        rel_tol_theta, rel_tol_L1 = self._extract_tolerances(deepcopy(tolerances))
         theta_errors = self._assert_theta_parameters_match_expected(params_table, rel_tol_theta)
         L1_errors = self._assert_L1_parameters_match_expected(params_table, rel_tol_L1)
 

From c18460a47ed6e3e464ebcacba6373fd7063d37a8 Mon Sep 17 00:00:00 2001
From: MialLewis <95620982+MialLewis@users.noreply.github.com>
Date: Thu, 27 Apr 2023 09:16:47 +0100
Subject: [PATCH 03/15] refactor invalid peaks

---
 .../calibrate_vesuvio_analysis.py             | 41 +++++++++++--------
 1 file changed, 23 insertions(+), 18 deletions(-)

diff --git a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
index df11daca..fb0067ce 100644
--- a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
+++ b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
@@ -250,6 +250,27 @@ def _calculate_final_flight_path(self, peak_table, spec_list):
         theta = EVSMiscFunctions.read_table_column(self._current_workspace, 'theta', spec_list)
         r_theta = EVSMiscFunctions.calculate_r_theta(self._sample_mass, theta)
 
+        peak_centres = self._filter_peak_centres_for_invalid_detectors(spec_list, peak_table)
+
+        delta_t = (peak_centres - t0) / 1e+6
+        delta_t_error = t0_error / 1e+6
+
+        E1 *= EVSGlobals.MEV_CONVERSION
+        v1 = np.sqrt( 2 * E1 /scipy.constants.m_n)
+        L1 = v1 * delta_t - L0 * r_theta
+        L1_error = v1 * delta_t_error
+
+        self._set_table_column(self._current_workspace, 'L1', L1, spec_list)
+        self._set_table_column(self._current_workspace, 'L1_Err', L1_error, spec_list)
+
+    def _filter_peak_centres_for_invalid_detectors(self, spec_list, peak_table):
+        """
+          Finds invalid detectors and filters the peak centres. Caches the invalid detectors found to avoid repeat identification.
+          @param spec_list detectors to consider, must be either FRONT or BACKSCATTERING range.
+          @param peak_table - name of table containing fitted parameters each spectra.
+
+          @returns peak_centres - a list of peak fitted peak centres, with those that represent invalid detectors marked nan.
+        """
         peak_centres = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzPos', spec_list)
         peak_centres_errors = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzPos_Err', spec_list)
 
@@ -267,20 +288,10 @@ def _calculate_final_flight_path(self, peak_table, spec_list):
             raise AttributeError("Spec list invalid - must represent either front or back detectors.")
 
         peak_centres[invalid_detectors] = np.nan
-
         print(f'Invalid Spectra Index Found and Marked NAN: {invalid_detectors} from Spectra Index List:'
               f'{[ x -3 for x in spec_list]}')
 
-        delta_t = (peak_centres - t0) / 1e+6
-        delta_t_error = t0_error / 1e+6
-
-        E1 *= EVSGlobals.MEV_CONVERSION
-        v1 = np.sqrt( 2 * E1 /scipy.constants.m_n)
-        L1 = v1 * delta_t - L0 * r_theta
-        L1_error = v1 * delta_t_error
-
-        self._set_table_column(self._current_workspace, 'L1', L1, spec_list)
-        self._set_table_column(self._current_workspace, 'L1_Err', L1_error, spec_list)
+        return peak_centres
 
     def _calculate_scattering_angle(self, table_name, spec_list):
         """
@@ -346,13 +357,7 @@ def _calculate_final_energy(self, peak_table, spec_list, calculate_global):
             theta = EVSMiscFunctions.read_table_column(self._current_workspace, 'theta', spec_list)
             r_theta = EVSMiscFunctions.calculate_r_theta(self._sample_mass, theta)
 
-            peak_centres = EVSMiscFunctions.read_fitting_result_table_column(peak_table[0], 'f1.LorentzPos', spec_list)
-            peak_centres_errors = EVSMiscFunctions.read_fitting_result_table_column(peak_table[0], 'f1.LorentzPos_Err', spec_list)
-
-            if not self._invalid_detectors_back.any():
-                self._invalid_detectors_back = EVSMiscFunctions.identify_invalid_spectra(peak_table[0], peak_centres, peak_centres_errors,
-                                                                                         spec_list)
-            peak_centres[self._invalid_detectors_back] = np.nan
+            peak_centres = self._filter_peak_centres_for_invalid_detectors(spec_list, peak_table[0])
 
             delta_t = (peak_centres - t0) / 1e+6
             v1 = (L0 * r_theta + L1) / delta_t

From 4001789aca24a11c2433085fafb19c12e7fac73e Mon Sep 17 00:00:00 2001
From: MialLewis <95620982+MialLewis@users.noreply.github.com>
Date: Thu, 27 Apr 2023 09:18:51 +0100
Subject: [PATCH 04/15] improve doc for InvalidDetectors prop

---
 .../calibration_scripts/calibrate_vesuvio_analysis.py           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
index fb0067ce..57e8e397 100644
--- a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
+++ b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
@@ -48,7 +48,7 @@ def PyInit(self):
         detector_validator.setLower(EVSGlobals.DETECTOR_RANGE[0])
         detector_validator.setUpper(EVSGlobals.DETECTOR_RANGE[-1])
         self.declareProperty(IntArrayProperty('InvalidDetectors', [], detector_validator, Direction.Input),
-                             doc="Detectors to be marked as invalid, to be excused from analysis calculations.")
+                             doc="List of detectors to be marked as invalid (3-198), to be excluded from analysis calculations.")
 
         self.declareProperty('Iterations', 2, validator=IntBoundedValidator(lower=1),
                              doc="Number of iterations to perform. Default is 2.")

From 4fdef0f080bc320b76e546690bd82ad84c223590 Mon Sep 17 00:00:00 2001
From: MialLewis <95620982+MialLewis@users.noreply.github.com>
Date: Thu, 27 Apr 2023 10:01:26 +0100
Subject: [PATCH 05/15] refactor invalid detectors into helper class

---
 .../calibrate_vesuvio_analysis.py             | 41 +-------
 .../calibrate_vesuvio_helper_functions.py     | 98 +++++++++++++------
 2 files changed, 73 insertions(+), 66 deletions(-)

diff --git a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
index 57e8e397..0045bc26 100644
--- a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
+++ b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
@@ -3,7 +3,7 @@
 from mantid.api import FileProperty, FileAction, PythonAlgorithm,AlgorithmManager
 from mantid.simpleapi import CreateEmptyTableWorkspace, DeleteWorkspace, ReplaceSpecialValues, GroupWorkspaces, mtd,\
      ConvertTableToMatrixWorkspace, ConjoinWorkspaces, Transpose, PlotPeakByLogValue,RenameWorkspace
-from calibration_scripts.calibrate_vesuvio_helper_functions import EVSGlobals, EVSMiscFunctions
+from calibration_scripts.calibrate_vesuvio_helper_functions import EVSGlobals, EVSMiscFunctions, InvalidDetectors
 
 
 import os
@@ -250,7 +250,7 @@ def _calculate_final_flight_path(self, peak_table, spec_list):
         theta = EVSMiscFunctions.read_table_column(self._current_workspace, 'theta', spec_list)
         r_theta = EVSMiscFunctions.calculate_r_theta(self._sample_mass, theta)
 
-        peak_centres = self._filter_peak_centres_for_invalid_detectors(spec_list, peak_table)
+        peak_centres = self._invalid_detectors.filter_peak_centres_for_invalid_detectors(spec_list, peak_table)
 
         delta_t = (peak_centres - t0) / 1e+6
         delta_t_error = t0_error / 1e+6
@@ -263,36 +263,6 @@ def _calculate_final_flight_path(self, peak_table, spec_list):
         self._set_table_column(self._current_workspace, 'L1', L1, spec_list)
         self._set_table_column(self._current_workspace, 'L1_Err', L1_error, spec_list)
 
-    def _filter_peak_centres_for_invalid_detectors(self, spec_list, peak_table):
-        """
-          Finds invalid detectors and filters the peak centres. Caches the invalid detectors found to avoid repeat identification.
-          @param spec_list detectors to consider, must be either FRONT or BACKSCATTERING range.
-          @param peak_table - name of table containing fitted parameters each spectra.
-
-          @returns peak_centres - a list of peak fitted peak centres, with those that represent invalid detectors marked nan.
-        """
-        peak_centres = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzPos', spec_list)
-        peak_centres_errors = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzPos_Err', spec_list)
-
-        if spec_list == EVSGlobals.FRONTSCATTERING_RANGE:
-            if not self._invalid_detectors_front.any():
-                self._invalid_detectors_front = EVSMiscFunctions.identify_invalid_spectra(peak_table, peak_centres, peak_centres_errors,
-                                                                                          spec_list)
-            invalid_detectors = self._invalid_detectors_front
-        elif spec_list == EVSGlobals.BACKSCATTERING_RANGE:
-            if not self._invalid_detectors_back.any():
-                self._invalid_detectors_back = EVSMiscFunctions.identify_invalid_spectra(peak_table, peak_centres, peak_centres_errors,
-                                                                                         spec_list)
-            invalid_detectors = self._invalid_detectors_back
-        else:
-            raise AttributeError("Spec list invalid - must represent either front or back detectors.")
-
-        peak_centres[invalid_detectors] = np.nan
-        print(f'Invalid Spectra Index Found and Marked NAN: {invalid_detectors} from Spectra Index List:'
-              f'{[ x -3 for x in spec_list]}')
-
-        return peak_centres
-
     def _calculate_scattering_angle(self, table_name, spec_list):
         """
           Calculate the total scattering angle using the previous calculated parameters.
@@ -357,7 +327,7 @@ def _calculate_final_energy(self, peak_table, spec_list, calculate_global):
             theta = EVSMiscFunctions.read_table_column(self._current_workspace, 'theta', spec_list)
             r_theta = EVSMiscFunctions.calculate_r_theta(self._sample_mass, theta)
 
-            peak_centres = self._filter_peak_centres_for_invalid_detectors(spec_list, peak_table[0])
+            peak_centres = self._invalid_detectors.filter_peak_centres_for_invalid_detectors(spec_list, peak_table[0])
 
             delta_t = (peak_centres - t0) / 1e+6
             v1 = (L0 * r_theta + L1) / delta_t
@@ -406,10 +376,7 @@ def _setup(self):
         self._sample_mass = self.getProperty("Mass").value
         self._d_spacings = self.getProperty("DSpacings").value.tolist()
         self._E1_value_and_error = self.getProperty("E1FixedValueAndError").value.tolist()
-        self._invalid_detectors_front = np.array([[x - EVSGlobals.FRONTSCATTERING_RANGE[0]] for x in self.getProperty("InvalidDetectors").value.tolist()
-                                                  if EVSGlobals.FRONTSCATTERING_RANGE[0] <= x <= EVSGlobals.FRONTSCATTERING_RANGE[-1]])
-        self._invalid_detectors_back = np.array([[x - EVSGlobals.BACKSCATTERING_RANGE[0]] for x in self.getProperty("InvalidDetectors").value.tolist()
-                                                if EVSGlobals.BACKSCATTERING_RANGE[0] <= x <= EVSGlobals.BACKSCATTERING_RANGE[-1]])
+        self._invalid_detectors = InvalidDetectors(self.getProperty("InvalidDetectors").value.tolist())
         self._shared_parameter_fit_type = self.getProperty("SharedParameterFitType").value
         self._calc_L0 = self.getProperty("CalculateL0").value
         self._make_IP_file = self.getProperty("CreateIPFile").value
diff --git a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py
index 964fc42c..05759154 100644
--- a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py
+++ b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py
@@ -92,35 +92,6 @@ def calculate_r_theta(sample_mass, thetas):
 
         return r_theta
 
-    @staticmethod
-    def identify_invalid_spectra(peak_table, peak_centres, peak_centres_errors, spec_list):
-        """
-          Inspect fitting results, and identify the fits associated with invalid spectra. These are spectra associated with detectors
-          which have lost foil coverage following a recent reduction in distance from source to detectors.
-
-          @param peak_table - name of table containing fitted parameters each spectra.
-          @param spec_list - spectrum range to inspect.
-          @return a list of invalid spectra.
-        """
-        peak_Gaussian_FWHM = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.GaussianFWHM', spec_list)
-        peak_Gaussian_FWHM_errors = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.GaussianFWHM_Err', spec_list)
-        peak_Lorentz_FWHM = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzFWHM', spec_list)
-        peak_Lorentz_FWHM_errors = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzFWHM_Err', spec_list)
-        peak_Lorentz_Amp = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzAmp', spec_list)
-        peak_Lorentz_Amp_errors = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzAmp_Err', spec_list)
-
-        invalid_spectra = np.argwhere((np.isinf(peak_Lorentz_Amp_errors)) | (np.isnan(peak_Lorentz_Amp_errors)) | \
-                                      (np.isinf(peak_centres_errors)) | (np.isnan(peak_centres_errors)) | \
-                                      (np.isnan(peak_Gaussian_FWHM_errors)) | (np.isinf(peak_Gaussian_FWHM_errors)) | \
-                                      (np.isnan(peak_Lorentz_FWHM_errors)) | (np.isinf(peak_Lorentz_FWHM_errors)) | \
-                                      (np.isnan(peak_Lorentz_Amp_errors)) | (np.isinf(peak_Lorentz_Amp_errors)) | \
-                                      (np.absolute(peak_Gaussian_FWHM_errors) > np.absolute(peak_Gaussian_FWHM)) | \
-                                      (np.absolute(peak_Lorentz_FWHM_errors) > np.absolute(peak_Lorentz_FWHM)) | \
-                                      (np.absolute(peak_Lorentz_Amp_errors) > np.absolute(peak_Lorentz_Amp)) | \
-                                      (np.absolute(peak_centres_errors) > np.absolute(peak_centres)))
-
-        return invalid_spectra
-
     # The IP text file load function skips the first 3 rows of the text file.
     # This assumes that there is one line for the file header and 2 lines for
     # parameters of monitor 1 and monitor 2, respectively.
@@ -216,3 +187,72 @@ def generate_fit_function_header(function_type, error=False):
             raise ValueError("Unsupported fit function type: %s" % function_type)
 
         return {k: v + error_str for k, v in func_header.items()}
+
+
+class InvalidDetectors:
+
+    def __init__(self, invalid_detector_list):
+        self._invalid_detectors_front = self._preset_invalid_detectors(invalid_detector_list, EVSGlobals.FRONTSCATTERING_RANGE)
+        self._invalid_detectors_back = self._preset_invalid_detectors(invalid_detector_list, EVSGlobals.BACKSCATTERING_RANGE)
+
+    @staticmethod
+    def _preset_invalid_detectors(invalid_detector_list_full_range, desired_range):
+        return np.array([[x - desired_range[0]] for x in invalid_detector_list_full_range if desired_range[0] <= x <= desired_range[-1]])
+
+    def filter_peak_centres_for_invalid_detectors(self, spec_list, peak_table):
+        """
+          Finds invalid detectors and filters the peak centres. Caches the invalid detectors found to avoid repeat identification.
+          @param spec_list detectors to consider, must be either FRONT or BACKSCATTERING range.
+          @param peak_table - name of table containing fitted parameters each spectra.
+
+          @returns peak_centres - a list of peak fitted peak centres, with those that represent invalid detectors marked nan.
+        """
+        peak_centres = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzPos', spec_list)
+        peak_centres_errors = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzPos_Err', spec_list)
+
+        if spec_list == EVSGlobals.FRONTSCATTERING_RANGE:
+            if not self._invalid_detectors_front.any():
+                self._invalid_detectors_front = self._identify_invalid_spectra(peak_table, peak_centres, peak_centres_errors,
+                                                                               spec_list)
+            invalid_detectors = self._invalid_detectors_front
+        elif spec_list == EVSGlobals.BACKSCATTERING_RANGE:
+            if not self._invalid_detectors_back.any():
+                self._invalid_detectors_back = self._identify_invalid_spectra(peak_table, peak_centres, peak_centres_errors,
+                                                                              spec_list)
+            invalid_detectors = self._invalid_detectors_back
+        else:
+            raise AttributeError("Spec list invalid - must represent either front or back detectors.")
+
+        peak_centres[invalid_detectors] = np.nan
+        print(f'Invalid Spectra Index Found and Marked NAN: {invalid_detectors} from Spectra Index List:'
+              f'{[ x -3 for x in spec_list]}')
+
+        return peak_centres
+
+    @staticmethod
+    def _identify_invalid_spectra(peak_table, peak_centres, peak_centres_errors, spec_list):
+        """
+          Inspect fitting results, and identify the fits associated with invalid spectra. These are spectra associated with detectors
+          which have lost foil coverage following a recent reduction in distance from source to detectors.
+
+          @param peak_table - name of table containing fitted parameters each spectra.
+          @param spec_list - spectrum range to inspect.
+          @return a list of invalid spectra.
+        """
+        peak_Gaussian_FWHM = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.GaussianFWHM', spec_list)
+        peak_Gaussian_FWHM_errors = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.GaussianFWHM_Err', spec_list)
+        peak_Lorentz_FWHM = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzFWHM', spec_list)
+        peak_Lorentz_FWHM_errors = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzFWHM_Err', spec_list)
+        peak_Lorentz_Amp = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzAmp', spec_list)
+        peak_Lorentz_Amp_errors = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzAmp_Err', spec_list)
+
+        invalid_spectra = np.argwhere((np.isinf(peak_Lorentz_Amp_errors)) | (np.isnan(peak_Lorentz_Amp_errors)) | \
+                                      (np.isinf(peak_centres_errors)) | (np.isnan(peak_centres_errors)) | \
+                                      (np.isnan(peak_Gaussian_FWHM_errors)) | (np.isinf(peak_Gaussian_FWHM_errors)) | \
+                                      (np.isnan(peak_Lorentz_FWHM_errors)) | (np.isinf(peak_Lorentz_FWHM_errors)) | \
+                                      (np.isnan(peak_Lorentz_Amp_errors)) | (np.isinf(peak_Lorentz_Amp_errors)) | \
+                                      (np.absolute(peak_Gaussian_FWHM_errors) > np.absolute(peak_Gaussian_FWHM)) | \
+                                      (np.absolute(peak_Lorentz_FWHM_errors) > np.absolute(peak_Lorentz_FWHM)) | \
+                                      (np.absolute(peak_Lorentz_Amp_errors) > np.absolute(peak_Lorentz_Amp)) | \
+                                      (np.absolute(peak_centres_errors) > np.absolute(peak_centres)))
+        return invalid_spectra
\ No newline at end of file

From 3e2f94f45562709135eae020037b200fd3221560 Mon Sep 17 00:00:00 2001
From: MialLewis <95620982+MialLewis@users.noreply.github.com>
Date: Thu, 27 Apr 2023 15:09:02 +0100
Subject: [PATCH 06/15] temp unfinished commit to rebase

---
 .../calibrate_vesuvio_analysis.py             | 21 +++++++++---
 .../calibrate_vesuvio_fit.py                  | 32 +++++++++++++++----
 .../calibrate_vesuvio_helper_functions.py     |  4 +++
 3 files changed, 46 insertions(+), 11 deletions(-)

diff --git a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
index 0045bc26..f3014c22 100644
--- a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
+++ b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
@@ -1,8 +1,8 @@
 from mantid.kernel import StringArrayProperty, Direction, StringListValidator, FloatArrayBoundedValidator, StringMandatoryValidator,\
      IntBoundedValidator, FloatArrayProperty
-from mantid.api import FileProperty, FileAction, PythonAlgorithm,AlgorithmManager
+from mantid.api import FileProperty, FileAction, PythonAlgorithm, AlgorithmManager
 from mantid.simpleapi import CreateEmptyTableWorkspace, DeleteWorkspace, ReplaceSpecialValues, GroupWorkspaces, mtd,\
-     ConvertTableToMatrixWorkspace, ConjoinWorkspaces, Transpose, PlotPeakByLogValue,RenameWorkspace
+     ConvertTableToMatrixWorkspace, ConjoinWorkspaces, Transpose, PlotPeakByLogValue, RenameWorkspace
 from calibration_scripts.calibrate_vesuvio_helper_functions import EVSGlobals, EVSMiscFunctions, InvalidDetectors
 
 
@@ -128,10 +128,18 @@ def PyExec(self):
 
             # calibrate  E1 for backscattering detectors and use the backscattering averaged value for all detectors
             E1_fit_back = self._current_workspace + '_E1_back'
+            invalid_detectors = \
             self._run_calibration_fit(Samples=self._samples, Function='Voigt', Mode='SingleDifference',
                                       SpectrumRange=EVSGlobals.BACKSCATTERING_RANGE, InstrumentParameterWorkspace=self._param_table,
                                       Mass=self._sample_mass, OutputWorkspace=E1_fit_back, CreateOutput=self._create_output,
-                                      PeakType='Recoil', SharedParameterFitType=self._shared_parameter_fit_type)
+                                      PeakType='Recoil', SharedParameterFitType=self._shared_parameter_fit_type,
+                                      InvalidDetectors=self._invalid_detectors.get_all_invalid_detectors())
+
+            import pydevd_pycharm
+            pydevd_pycharm.settrace('localhost', port=8080, stdoutToServer=True, stderrToServer=True)
+
+            if invalid_detectors and not self._invalid_detectors.get_all_invalid_detectors():
+                self._invalid_detectors = InvalidDetectors(invalid_detectors)
 
             E1_peak_fits_back = mtd[self._current_workspace + '_E1_back_Peak_Parameters'].getNames()
             self._calculate_final_energy(E1_peak_fits_back, EVSGlobals.BACKSCATTERING_RANGE, self._shared_parameter_fit_type != "Individual")
@@ -144,7 +152,8 @@ def PyExec(self):
             self._run_calibration_fit(Samples=self._samples, Function='Voigt', Mode='SingleDifference',
                                       SpectrumRange=EVSGlobals.FRONTSCATTERING_RANGE, InstrumentParameterWorkspace=self._param_table,
                                       Mass=self._sample_mass, OutputWorkspace=E1_fit_front, CreateOutput=self._create_output,
-                                      PeakType='Recoil', SharedParameterFitType=self._shared_parameter_fit_type)
+                                      PeakType='Recoil', SharedParameterFitType=self._shared_parameter_fit_type,
+                                      InvalidDetectors=self._invalid_detectors.get_all_invalid_detectors())
 
             E1_peak_fits_front = mtd[self._current_workspace + '_E1_front_Peak_Parameters'].getNames()
             self._calculate_final_flight_path(E1_peak_fits_front[0], EVSGlobals.FRONTSCATTERING_RANGE)
@@ -183,14 +192,18 @@ def _run_calibration_fit(self, *args, **kwargs):
 
           @param args - positional arguments to the algorithm
           @param kwargs - key word arguments to the algorithm
+          @returns - returns list of invalid detectors so set in overarching alg
         """
         from mantid.simpleapi import set_properties
         alg = AlgorithmManager.create('EVSCalibrationFit')
         alg.initialize()
         alg.setRethrows(True)
+
         set_properties(alg, *args, **kwargs)
         alg.execute()
 
+        return alg.getProperty("InvalidDetectors").value.tolist()
+
     def _calculate_time_delay(self, table_name, spec_list):
         """
           Calculate time delay from frontscattering detectors.
diff --git a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_fit.py b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_fit.py
index 1fbe846b..9fc03674 100644
--- a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_fit.py
+++ b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_fit.py
@@ -6,7 +6,7 @@
      ReplaceSpecialValues, FindPeaks, GroupWorkspaces, mtd, Plus, LoadVesuvio, LoadRaw, ConvertToDistribution, FindPeakBackground,\
      ExtractSingleSpectrum, SumSpectra, AppendSpectra, CloneWorkspace, Fit, MaskDetectors, ExtractUnmaskedSpectra, CreateWorkspace
 from functools import partial
-from calibration_scripts.calibrate_vesuvio_helper_functions import EVSGlobals, EVSMiscFunctions
+from calibration_scripts.calibrate_vesuvio_helper_functions import EVSGlobals, EVSMiscFunctions, InvalidDetectors
 
 import os
 import sys
@@ -68,6 +68,14 @@ def PyInit(self):
                              doc='Calculate shared parameters using an individual and/or'
                                  'global fit.', validator=shared_fit_type_validator)
 
+        detector_validator = IntArrayBoundedValidator()
+        detector_validator.setLower(EVSGlobals.DETECTOR_RANGE[0])
+        detector_validator.setUpper(EVSGlobals.DETECTOR_RANGE[-1])
+        self.declareProperty(IntArrayProperty('InvalidDetectors', [], detector_validator, Direction.Input),
+                             doc="List of detectors to be marked as invalid (3-198), to be excluded from analysis calculations.")
+        self.declareProperty(IntArrayProperty('InvalidDetectorsOut', [], detector_validator, Direction.Output),
+                             doc="List of detectors found as invalid to be output.")
+
         self.declareProperty(
             ITableWorkspaceProperty("InstrumentParameterWorkspace", "", Direction.Input, PropertyMode.Optional),
             doc='Workspace contain instrument parameters.')
@@ -91,6 +99,9 @@ def PyExec(self):
         if self._create_output and not self._fitting_bragg_peaks:
             self._generate_fit_workspaces()
 
+        # set invalid detectors output param
+        self.setProperty("InvalidDetectorsOut", self._invalid_detectors.get_all_invalid_detectors())
+
         # clean up workspaces
         if self._param_file != "":
             DeleteWorkspace(self._param_table)
@@ -112,6 +123,7 @@ def _setup_class_variables_from_properties(self):
         self._sample_mass = self.getProperty("Mass").value
         self._create_output = self.getProperty("CreateOutput").value
         self._shared_parameter_fit_type = self.getProperty("SharedParameterFitType").value
+        self._invalid_detectors = InvalidDetectors(self.getProperty("InvalidDetectors").value.tolist())
 
     def _setup_spectra_list(self):
         self._spec_list = self.getProperty("SpectrumRange").value.tolist()
@@ -549,6 +561,10 @@ def _fit_peaks(self):
 
         self._output_parameter_tables = []
         self._peak_fit_workspaces = []
+
+        import pydevd_pycharm
+        pydevd_pycharm.settrace('localhost', port=8080, stdoutToServer=True, stderrToServer=True)
+
         for peak_index, estimated_peak_positions in enumerate(estimated_peak_positions_all_peaks):
 
             self._peak_fit_workspaces_by_spec = []
@@ -585,12 +601,14 @@ def _shared_parameter_fit(self, output_parameter_table_name, output_parameter_ta
         initial_params = {'A0': 0.0, 'A1': 0.0, 'LorentzAmp': init_Lorentz_Amp, 'LorentzPos': init_Lorentz_Pos,
                           'LorentzFWHM': init_Lorentz_FWHM, 'GaussianFWHM': init_Gaussian_FWHM}
 
-        peak_centres = EVSMiscFunctions.read_fitting_result_table_column(output_parameter_table_name, 'f1.LorentzPos', self._spec_list)
-        peak_centres_errors = EVSMiscFunctions.read_fitting_result_table_column(output_parameter_table_name, 'f1.LorentzPos_Err',
-                                                                                self._spec_list)
-        invalid_spectra = EVSMiscFunctions.identify_invalid_spectra(output_parameter_table_name, peak_centres, peak_centres_errors,
-                                                                    self._spec_list)
-        self._fit_shared_parameter(invalid_spectra, initial_params, output_parameter_table_headers)
+        import pydevd_pycharm
+        pydevd_pycharm.settrace('localhost', port=8080, stdoutToServer=True, stderrToServer=True)
+
+        #This caches all the invalid spectra within self._invalid_detectors
+        self._invalid_detectors.filter_peak_centres_for_invalid_detectors(self._spec_list, output_parameter_table_name)
+
+        self._fit_shared_parameter([[x] for x in self._invalid_detectors.get_all_invalid_detectors()], initial_params,
+                                   output_parameter_table_headers)
 
     def _fit_peak(self, peak_index, spec_index, peak_position, output_parameter_table_name,
                   output_parameter_table_headers):
diff --git a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py
index 05759154..a7fc612c 100644
--- a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py
+++ b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py
@@ -192,9 +192,13 @@ def generate_fit_function_header(function_type, error=False):
 class InvalidDetectors:
 
     def __init__(self, invalid_detector_list):
+        self._invalid_detectors = invalid_detector_list
         self._invalid_detectors_front = self._preset_invalid_detectors(invalid_detector_list, EVSGlobals.FRONTSCATTERING_RANGE)
         self._invalid_detectors_back = self._preset_invalid_detectors(invalid_detector_list, EVSGlobals.BACKSCATTERING_RANGE)
 
+    def get_all_invalid_detectors(self):
+        return self._invalid_detectors
+
     @staticmethod
     def _preset_invalid_detectors(invalid_detector_list_full_range, desired_range):
         return np.array([[x - desired_range[0]] for x in invalid_detector_list_full_range if desired_range[0] <= x <= desired_range[-1]])

From 0d05bef77c23131ae3b0e53eeac3b44b8b1ec636 Mon Sep 17 00:00:00 2001
From: MialLewis <95620982+MialLewis@users.noreply.github.com>
Date: Fri, 28 Apr 2023 16:46:39 +0100
Subject: [PATCH 07/15] add invalid detector feedback from fit class

---
 .../calibrate_vesuvio_analysis.py             | 11 +--
 .../calibrate_vesuvio_fit.py                  |  8 +-
 .../calibrate_vesuvio_helper_functions.py     | 24 +++++-
 .../tests/system/test_system_fit.py           | 16 ++++
 .../tests/unit/test_calibrate_vesuvio_misc.py | 75 ++++++++++++++++++-
 5 files changed, 116 insertions(+), 18 deletions(-)

diff --git a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
index f3014c22..34d1933e 100644
--- a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
+++ b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
@@ -134,12 +134,7 @@ def PyExec(self):
                                       Mass=self._sample_mass, OutputWorkspace=E1_fit_back, CreateOutput=self._create_output,
                                       PeakType='Recoil', SharedParameterFitType=self._shared_parameter_fit_type,
                                       InvalidDetectors=self._invalid_detectors.get_all_invalid_detectors())
-
-            import pydevd_pycharm
-            pydevd_pycharm.settrace('localhost', port=8080, stdoutToServer=True, stderrToServer=True)
-
-            if invalid_detectors and not self._invalid_detectors.get_all_invalid_detectors():
-                self._invalid_detectors = InvalidDetectors(invalid_detectors)
+            self._invalid_detectors.add_invalid_detectors(invalid_detectors)
 
             E1_peak_fits_back = mtd[self._current_workspace + '_E1_back_Peak_Parameters'].getNames()
             self._calculate_final_energy(E1_peak_fits_back, EVSGlobals.BACKSCATTERING_RANGE, self._shared_parameter_fit_type != "Individual")
@@ -149,11 +144,13 @@ def PyExec(self):
 
             # calibrate L1 for frontscattering detectors based on the averaged E1 value  and calibrated theta values
             E1_fit_front = self._current_workspace + '_E1_front'
+            invalid_detectors += \
             self._run_calibration_fit(Samples=self._samples, Function='Voigt', Mode='SingleDifference',
                                       SpectrumRange=EVSGlobals.FRONTSCATTERING_RANGE, InstrumentParameterWorkspace=self._param_table,
                                       Mass=self._sample_mass, OutputWorkspace=E1_fit_front, CreateOutput=self._create_output,
                                       PeakType='Recoil', SharedParameterFitType=self._shared_parameter_fit_type,
                                       InvalidDetectors=self._invalid_detectors.get_all_invalid_detectors())
+            self._invalid_detectors.add_invalid_detectors(invalid_detectors)
 
             E1_peak_fits_front = mtd[self._current_workspace + '_E1_front_Peak_Parameters'].getNames()
             self._calculate_final_flight_path(E1_peak_fits_front[0], EVSGlobals.FRONTSCATTERING_RANGE)
@@ -202,7 +199,7 @@ def _run_calibration_fit(self, *args, **kwargs):
         set_properties(alg, *args, **kwargs)
         alg.execute()
 
-        return alg.getProperty("InvalidDetectors").value.tolist()
+        return alg.getProperty("InvalidDetectorsOut").value.tolist()
 
     def _calculate_time_delay(self, table_name, spec_list):
         """
diff --git a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_fit.py b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_fit.py
index 9fc03674..7ec11c6f 100644
--- a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_fit.py
+++ b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_fit.py
@@ -562,9 +562,6 @@ def _fit_peaks(self):
         self._output_parameter_tables = []
         self._peak_fit_workspaces = []
 
-        import pydevd_pycharm
-        pydevd_pycharm.settrace('localhost', port=8080, stdoutToServer=True, stderrToServer=True)
-
         for peak_index, estimated_peak_positions in enumerate(estimated_peak_positions_all_peaks):
 
             self._peak_fit_workspaces_by_spec = []
@@ -601,13 +598,10 @@ def _shared_parameter_fit(self, output_parameter_table_name, output_parameter_ta
         initial_params = {'A0': 0.0, 'A1': 0.0, 'LorentzAmp': init_Lorentz_Amp, 'LorentzPos': init_Lorentz_Pos,
                           'LorentzFWHM': init_Lorentz_FWHM, 'GaussianFWHM': init_Gaussian_FWHM}
 
-        import pydevd_pycharm
-        pydevd_pycharm.settrace('localhost', port=8080, stdoutToServer=True, stderrToServer=True)
-
         #This caches all the invalid spectra within self._invalid_detectors
         self._invalid_detectors.filter_peak_centres_for_invalid_detectors(self._spec_list, output_parameter_table_name)
 
-        self._fit_shared_parameter([[x] for x in self._invalid_detectors.get_all_invalid_detectors()], initial_params,
+        self._fit_shared_parameter(self._invalid_detectors.get_invalid_detectors_index(self._spec_list), initial_params,
                                    output_parameter_table_headers)
 
     def _fit_peak(self, peak_index, spec_index, peak_position, output_parameter_table_name,
diff --git a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py
index a7fc612c..c0620b43 100644
--- a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py
+++ b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py
@@ -192,12 +192,30 @@ def generate_fit_function_header(function_type, error=False):
 class InvalidDetectors:
 
     def __init__(self, invalid_detector_list):
-        self._invalid_detectors = invalid_detector_list
         self._invalid_detectors_front = self._preset_invalid_detectors(invalid_detector_list, EVSGlobals.FRONTSCATTERING_RANGE)
         self._invalid_detectors_back = self._preset_invalid_detectors(invalid_detector_list, EVSGlobals.BACKSCATTERING_RANGE)
 
+    def add_invalid_detectors(self, invalid_detector_list):
+        """
+          Takes a list of invalid spectra, adds unique entries to the stored np arrays.
+          @param invalid_detector_list - list of detectors to append to detector list, if unique.
+        """
+        self._invalid_detectors_front = np.array([[x] for x in sorted(set(self._invalid_detectors_front.ravel()).union(
+            set(self._preset_invalid_detectors(invalid_detector_list, EVSGlobals.FRONTSCATTERING_RANGE).ravel())))])
+        self._invalid_detectors_back = np.array([[x] for x in sorted(set(self._invalid_detectors_back.ravel()).union(
+            set(self._preset_invalid_detectors(invalid_detector_list, EVSGlobals.BACKSCATTERING_RANGE).ravel())))])
+
     def get_all_invalid_detectors(self):
-        return self._invalid_detectors
+        return [i + EVSGlobals.BACKSCATTERING_RANGE[0] for i in self._invalid_detectors_back.flatten().tolist()] + \
+               [j + EVSGlobals.FRONTSCATTERING_RANGE[0] for j in self._invalid_detectors_front.flatten().tolist()]
+
+    def get_invalid_detectors_index(self, desired_range):
+        if desired_range == EVSGlobals.FRONTSCATTERING_RANGE:
+            return self._invalid_detectors_front.flatten().tolist()
+        elif desired_range == EVSGlobals.BACKSCATTERING_RANGE:
+            return self._invalid_detectors_back.flatten().tolist()
+        else:
+            raise AttributeError("desired range invalid - must represent either front or back detectors.")
 
     @staticmethod
     def _preset_invalid_detectors(invalid_detector_list_full_range, desired_range):
@@ -229,7 +247,7 @@ def filter_peak_centres_for_invalid_detectors(self, spec_list, peak_table):
 
         peak_centres[invalid_detectors] = np.nan
         print(f'Invalid Spectra Index Found and Marked NAN: {invalid_detectors} from Spectra Index List:'
-              f'{[ x -3 for x in spec_list]}')
+              f'{[ x - EVSGlobals.DETECTOR_RANGE[0] for x in spec_list]}')
 
         return peak_centres
 
diff --git a/unpackaged/vesuvio_calibration/tests/system/test_system_fit.py b/unpackaged/vesuvio_calibration/tests/system/test_system_fit.py
index 0e4698ce..0f1c9d66 100644
--- a/unpackaged/vesuvio_calibration/tests/system/test_system_fit.py
+++ b/unpackaged/vesuvio_calibration/tests/system/test_system_fit.py
@@ -121,6 +121,22 @@ def test_fit_backscattering_uranium(self, load_file_mock):
         params_table = self._run_evs_calibration_fit("Recoil")
         self._assert_fitted_positions_match_expected(expected_values, params_table, default_rel_tol=0.01)
 
+    @patch('unpackaged.vesuvio_calibration.calibration_scripts.calibrate_vesuvio_fit.EVSCalibrationFit._load_file')
+    def test_fit_peaks_copper_E1_invalid_detectors_specified(self, load_file_mock):
+        self._setup_copper_test()
+        self._E1_fit_active = True
+        self._E1_fit = [True]
+        self._output_workspace = "copper_peak_fit"
+        self._mode = ['SingleDifference']
+
+        load_file_mock.side_effect = self._load_file_side_effect
+        expected_values = self._calculate_energy_peak_positions()
+
+        self._create_evs_calibration_fit("Recoil")
+        self._alg.setProperty('InvalidDetectors', [10, 20, 30])
+        params_table = self._execute_evs_calibration_fit()
+        self._assert_fitted_positions_match_expected(expected_values, params_table, {38:  0.12})
+
     def _assert_fitted_positions_match_expected(self, expected_positions, params_table, rel_tolerance=None,
                                                 default_rel_tol=TestConstants.DEFAULT_RELATIVE_TOLERANCE):
         """
diff --git a/unpackaged/vesuvio_calibration/tests/unit/test_calibrate_vesuvio_misc.py b/unpackaged/vesuvio_calibration/tests/unit/test_calibrate_vesuvio_misc.py
index d995dff3..3a4a5c76 100644
--- a/unpackaged/vesuvio_calibration/tests/unit/test_calibrate_vesuvio_misc.py
+++ b/unpackaged/vesuvio_calibration/tests/unit/test_calibrate_vesuvio_misc.py
@@ -1,4 +1,4 @@
-from calibration_scripts.calibrate_vesuvio_helper_functions import EVSMiscFunctions
+from calibration_scripts.calibrate_vesuvio_helper_functions import EVSMiscFunctions, InvalidDetectors
 from mock import MagicMock, patch
 
 import unittest
@@ -82,6 +82,79 @@ def test_identify_invalid_spectra_error_greater_than_peak(self, mock_mtd):
         np.testing.assert_equal(np.argwhere(np.array([False, True, True])),
                                 EVSMiscFunctions.identify_invalid_spectra('peak_table', [5, 10, 20], [0.1, 0.15, 0.2], [0, 2]))
 
+    def test_create_empty_invalid_detectors(self):
+        invalid_detectors = InvalidDetectors([])
+        self.assertEqual(invalid_detectors._invalid_detectors_back.tolist(), [])
+        self.assertEqual(invalid_detectors._invalid_detectors_front.tolist(), [])
+
+    def test_create_invalid_detectors_back(self):
+        invalid_detectors = InvalidDetectors([10, 20, 30])
+        self.assertEqual(invalid_detectors._invalid_detectors_back.tolist(), [[7], [17], [27]])
+        self.assertEqual(invalid_detectors._invalid_detectors_front.tolist(), [])
+
+    def test_create_invalid_detectors_front(self):
+        invalid_detectors = InvalidDetectors([150, 160, 170])
+        self.assertEqual(invalid_detectors._invalid_detectors_back.tolist(), [])
+        self.assertEqual(invalid_detectors._invalid_detectors_front.tolist(), [[15], [25], [35]])
+
+    def test_create_invalid_detectors(self):
+        invalid_detectors = InvalidDetectors([10, 20, 30, 150, 160, 170])
+        self.assertEqual(invalid_detectors._invalid_detectors_back.tolist(), [[7], [17], [27]])
+        self.assertEqual(invalid_detectors._invalid_detectors_front.tolist(), [[15], [25], [35]])
+
+    def test_get_all_detectors(self):
+        input_invalid_detectors = [10, 20, 30, 150, 160, 170]
+        invalid_detectors = InvalidDetectors([10, 20, 30, 150, 160, 170])
+        self.assertEqual(invalid_detectors.get_all_invalid_detectors(), input_invalid_detectors)
+
+    @patch('unpackaged.vesuvio_calibration.calibration_scripts.calibrate_vesuvio_helper_functions'
+           '.EVSMiscFunctions.read_fitting_result_table_column')
+    def test_filter_peak_centres_for_invalid_detectors_front(self, mock_read_fitting_result):
+        invalid_detectors = InvalidDetectors([10, 20, 30, 150, 160, 170])
+        peak_table = 'input_peak_table'
+        mock_read_fitting_result.return_value = np.array([[float(x)] for x in range(3, 198, 1)])
+
+        out_peak_centres = invalid_detectors.filter_peak_centres_for_invalid_detectors([3, 134], peak_table)
+        self.assertEqual(list(np.argwhere(np.isnan(out_peak_centres)).transpose()[0]), [7, 17, 27])
+
+    @patch('unpackaged.vesuvio_calibration.calibration_scripts.calibrate_vesuvio_helper_functions'
+           '.EVSMiscFunctions.read_fitting_result_table_column')
+    def test_filter_peak_centres_for_invalid_detectors_back(self, mock_read_fitting_result):
+        invalid_detectors = InvalidDetectors([10, 20, 30, 150, 160, 170])
+        peak_table = 'input_peak_table'
+        mock_read_fitting_result.return_value = np.array([[float(x)] for x in range(3, 198, 1)])
+
+        out_peak_centres = invalid_detectors.filter_peak_centres_for_invalid_detectors([135, 198], peak_table)
+        self.assertEqual(list(np.argwhere(np.isnan(out_peak_centres)).transpose()[0]), [15, 25, 35])
+
+    @patch('unpackaged.vesuvio_calibration.calibration_scripts.calibrate_vesuvio_helper_functions'
+           '.EVSMiscFunctions.read_fitting_result_table_column')
+    def test_filter_peak_centres_for_invalid_detectors_invalid_range(self, mock_read_fitting_result):
+        invalid_detectors = InvalidDetectors([10, 20, 30, 150, 160, 170])
+        peak_table = 'input_peak_table'
+        with self.assertRaises(AttributeError):
+            invalid_detectors.filter_peak_centres_for_invalid_detectors([10, 20], peak_table)
+
+    def test_get_invalid_detectors_index_back(self):
+        invalid_detectors = InvalidDetectors([10, 20, 30, 150, 160, 170])
+        invalid_detectors_index = invalid_detectors.get_invalid_detectors_index([3, 134])
+        self.assertEqual(invalid_detectors_index, [7, 17, 27])
+
+    def test_get_invalid_detectors_index_front(self):
+        invalid_detectors = InvalidDetectors([10, 20, 30, 150, 160, 170])
+        invalid_detectors_index = invalid_detectors.get_invalid_detectors_index([135, 198])
+        self.assertEqual(invalid_detectors_index, [15, 25, 35])
+
+    def test_get_invalid_detectors_index_invalid_range(self):
+        invalid_detectors = InvalidDetectors([10, 20, 30, 150, 160, 170])
+        with self.assertRaises(AttributeError):
+            invalid_detectors.get_invalid_detectors_index([10, 20])
+
+    def test_add_invalid_detectors(self):
+        invalid_detectors = InvalidDetectors([10, 20, 30, 150, 160, 170])
+        invalid_detectors.add_invalid_detectors([10, 20, 25, 30, 180, 190])
+        self.assertEqual(invalid_detectors.get_all_invalid_detectors(), [10, 20, 25, 30, 150, 160, 170, 180, 190])
+
 
 if __name__ == '__main__':
     unittest.main()

From b35c65d4d2d41dad0a62e074a8d5f5a2d982dce6 Mon Sep 17 00:00:00 2001
From: MialLewis <95620982+MialLewis@users.noreply.github.com>
Date: Fri, 28 Apr 2023 17:29:03 +0100
Subject: [PATCH 08/15] fix unit tests and slight refactor invalid detectors

---
 .../calibrate_vesuvio_fit.py                  |  3 +-
 .../calibrate_vesuvio_helper_functions.py     | 36 +++++++++++--------
 .../tests/unit/test_calibrate_vesuvio_misc.py |  8 ++---
 3 files changed, 27 insertions(+), 20 deletions(-)

diff --git a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_fit.py b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_fit.py
index 7ec11c6f..efef766a 100644
--- a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_fit.py
+++ b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_fit.py
@@ -598,8 +598,7 @@ def _shared_parameter_fit(self, output_parameter_table_name, output_parameter_ta
         initial_params = {'A0': 0.0, 'A1': 0.0, 'LorentzAmp': init_Lorentz_Amp, 'LorentzPos': init_Lorentz_Pos,
                           'LorentzFWHM': init_Lorentz_FWHM, 'GaussianFWHM': init_Gaussian_FWHM}
 
-        #This caches all the invalid spectra within self._invalid_detectors
-        self._invalid_detectors.filter_peak_centres_for_invalid_detectors(self._spec_list, output_parameter_table_name)
+        self._invalid_detectors.identify_and_set_invalid_detectors_from_range(self._spec_list, output_parameter_table_name)
 
         self._fit_shared_parameter(self._invalid_detectors.get_invalid_detectors_index(self._spec_list), initial_params,
                                    output_parameter_table_headers)
diff --git a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py
index c0620b43..215f6fd9 100644
--- a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py
+++ b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py
@@ -221,35 +221,43 @@ def get_invalid_detectors_index(self, desired_range):
     def _preset_invalid_detectors(invalid_detector_list_full_range, desired_range):
         return np.array([[x - desired_range[0]] for x in invalid_detector_list_full_range if desired_range[0] <= x <= desired_range[-1]])
 
-    def filter_peak_centres_for_invalid_detectors(self, spec_list, peak_table):
+    def filter_peak_centres_for_invalid_detectors(self, detector_range, peak_table):
         """
           Finds invalid detectors and filters the peak centres. Caches the invalid detectors found to avoid repeat identification.
-          @param spec_list detectors to consider, must be either FRONT or BACKSCATTERING range.
+          @param detector_range detectors to consider, must be either FRONT or BACKSCATTERING range.
           @param peak_table - name of table containing fitted parameters each spectra.
 
           @returns peak_centres - a list of peak fitted peak centres, with those that represent invalid detectors marked nan.
         """
-        peak_centres = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzPos', spec_list)
-        peak_centres_errors = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzPos_Err', spec_list)
 
-        if spec_list == EVSGlobals.FRONTSCATTERING_RANGE:
+        invalid_detectors = self.identify_and_set_invalid_detectors_from_range(detector_range, peak_table)
+        peak_centres = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzPos', detector_range)
+        peak_centres[invalid_detectors] = np.nan
+        return peak_centres
+
+    def identify_and_set_invalid_detectors_from_range(self, detector_range, peak_table):
+        peak_centres = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzPos', detector_range)
+        peak_centres_errors = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzPos_Err', detector_range)
+
+        if detector_range == EVSGlobals.FRONTSCATTERING_RANGE:
             if not self._invalid_detectors_front.any():
                 self._invalid_detectors_front = self._identify_invalid_spectra(peak_table, peak_centres, peak_centres_errors,
-                                                                               spec_list)
-            invalid_detectors = self._invalid_detectors_front
-        elif spec_list == EVSGlobals.BACKSCATTERING_RANGE:
+                                                                               detector_range)
+                self._print_invalid_detectors(self._invalid_detectors_front, detector_range)
+            return self._invalid_detectors_front
+        elif detector_range == EVSGlobals.BACKSCATTERING_RANGE:
             if not self._invalid_detectors_back.any():
                 self._invalid_detectors_back = self._identify_invalid_spectra(peak_table, peak_centres, peak_centres_errors,
-                                                                              spec_list)
-            invalid_detectors = self._invalid_detectors_back
+                                                                              detector_range)
+                self._print_invalid_detectors(self._invalid_detectors_back, detector_range)
+            return self._invalid_detectors_back
         else:
             raise AttributeError("Spec list invalid - must represent either front or back detectors.")
 
-        peak_centres[invalid_detectors] = np.nan
+    @staticmethod
+    def _print_invalid_detectors(invalid_detectors, detector_range):
         print(f'Invalid Spectra Index Found and Marked NAN: {invalid_detectors} from Spectra Index List:'
-              f'{[ x - EVSGlobals.DETECTOR_RANGE[0] for x in spec_list]}')
-
-        return peak_centres
+              f'{[x - EVSGlobals.DETECTOR_RANGE[0] for x in detector_range]}')
 
     @staticmethod
     def _identify_invalid_spectra(peak_table, peak_centres, peak_centres_errors, spec_list):
diff --git a/unpackaged/vesuvio_calibration/tests/unit/test_calibrate_vesuvio_misc.py b/unpackaged/vesuvio_calibration/tests/unit/test_calibrate_vesuvio_misc.py
index 3a4a5c76..ec4fc6a4 100644
--- a/unpackaged/vesuvio_calibration/tests/unit/test_calibrate_vesuvio_misc.py
+++ b/unpackaged/vesuvio_calibration/tests/unit/test_calibrate_vesuvio_misc.py
@@ -44,7 +44,7 @@ def test_identify_invalid_spectra_no_invalid(self, mock_mtd):
         mock_mtd.__getitem__.return_value = ws_mock
 
         np.testing.assert_equal(np.argwhere([]),
-                                EVSMiscFunctions.identify_invalid_spectra('peak_table', [5, 10, 20], [0.1, 0.15, 0.2], [0, 2]))
+                                InvalidDetectors._identify_invalid_spectra('peak_table', [5, 10, 20], [0.1, 0.15, 0.2], [0, 2]))
 
     @patch('calibration_scripts.calibrate_vesuvio_helper_functions.mtd')
     def test_identify_invalid_spectra_nan_in_errors(self, mock_mtd):
@@ -56,7 +56,7 @@ def test_identify_invalid_spectra_nan_in_errors(self, mock_mtd):
         mock_mtd.__getitem__.return_value = ws_mock
 
         np.testing.assert_equal(np.argwhere(np.array([True, False, True])),
-                                EVSMiscFunctions.identify_invalid_spectra('peak_table', [5, 10, 20], [0.1, 0.15, 0.2], [0, 2]))
+                                InvalidDetectors._identify_invalid_spectra('peak_table', [5, 10, 20], [0.1, 0.15, 0.2], [0, 2]))
 
     @patch('calibration_scripts.calibrate_vesuvio_helper_functions.mtd')
     def test_identify_invalid_spectra_inf_in_errors(self, mock_mtd):
@@ -68,7 +68,7 @@ def test_identify_invalid_spectra_inf_in_errors(self, mock_mtd):
         mock_mtd.__getitem__.return_value = ws_mock
 
         np.testing.assert_equal(np.argwhere(np.array([True, True, False])),
-                                EVSMiscFunctions.identify_invalid_spectra('peak_table', [5, 10, 20], [0.1, 0.15, 0.2], [0, 2]))
+                                InvalidDetectors._identify_invalid_spectra('peak_table', [5, 10, 20], [0.1, 0.15, 0.2], [0, 2]))
 
     @patch('calibration_scripts.calibrate_vesuvio_helper_functions.mtd')
     def test_identify_invalid_spectra_error_greater_than_peak(self, mock_mtd):
@@ -80,7 +80,7 @@ def test_identify_invalid_spectra_error_greater_than_peak(self, mock_mtd):
         mock_mtd.__getitem__.return_value = ws_mock
 
         np.testing.assert_equal(np.argwhere(np.array([False, True, True])),
-                                EVSMiscFunctions.identify_invalid_spectra('peak_table', [5, 10, 20], [0.1, 0.15, 0.2], [0, 2]))
+                                InvalidDetectors._identify_invalid_spectra('peak_table', [5, 10, 20], [0.1, 0.15, 0.2], [0, 2]))
 
     def test_create_empty_invalid_detectors(self):
         invalid_detectors = InvalidDetectors([])

From 56f356265f7d1798b39969ba80b0e4b27776fe6a Mon Sep 17 00:00:00 2001
From: MialLewis <95620982+MialLewis@users.noreply.github.com>
Date: Fri, 28 Apr 2023 17:33:16 +0100
Subject: [PATCH 09/15] update documentation

---
 .../calibrate_vesuvio_helper_functions.py                 | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py
index 215f6fd9..f2ff879d 100644
--- a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py
+++ b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py
@@ -236,6 +236,14 @@ def filter_peak_centres_for_invalid_detectors(self, detector_range, peak_table):
         return peak_centres
 
     def identify_and_set_invalid_detectors_from_range(self, detector_range, peak_table):
+        """
+          Finds invalid detectors and caches the invalid detectors. Will not look to calculate if invalid detectors already exist.
+          @param detector_range detectors to consider, must be either FRONT or BACKSCATTERING range.
+          @param peak_table - name of table containing fitted parameters each spectra.
+
+          @returns invalid_detectors - a list of the index's of invalid detector, in the context of the range they belong to.
+        """
+
         peak_centres = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzPos', detector_range)
         peak_centres_errors = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzPos_Err', detector_range)
 

From de9b8e760242110951fcb4404b63e370b80ca6b7 Mon Sep 17 00:00:00 2001
From: MialLewis <95620982+MialLewis@users.noreply.github.com>
Date: Fri, 28 Apr 2023 17:37:13 +0100
Subject: [PATCH 10/15] remove defunct unit test

---
 .../tests/system/test_system_fit.py              | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/unpackaged/vesuvio_calibration/tests/system/test_system_fit.py b/unpackaged/vesuvio_calibration/tests/system/test_system_fit.py
index 0f1c9d66..0e4698ce 100644
--- a/unpackaged/vesuvio_calibration/tests/system/test_system_fit.py
+++ b/unpackaged/vesuvio_calibration/tests/system/test_system_fit.py
@@ -121,22 +121,6 @@ def test_fit_backscattering_uranium(self, load_file_mock):
         params_table = self._run_evs_calibration_fit("Recoil")
         self._assert_fitted_positions_match_expected(expected_values, params_table, default_rel_tol=0.01)
 
-    @patch('unpackaged.vesuvio_calibration.calibration_scripts.calibrate_vesuvio_fit.EVSCalibrationFit._load_file')
-    def test_fit_peaks_copper_E1_invalid_detectors_specified(self, load_file_mock):
-        self._setup_copper_test()
-        self._E1_fit_active = True
-        self._E1_fit = [True]
-        self._output_workspace = "copper_peak_fit"
-        self._mode = ['SingleDifference']
-
-        load_file_mock.side_effect = self._load_file_side_effect
-        expected_values = self._calculate_energy_peak_positions()
-
-        self._create_evs_calibration_fit("Recoil")
-        self._alg.setProperty('InvalidDetectors', [10, 20, 30])
-        params_table = self._execute_evs_calibration_fit()
-        self._assert_fitted_positions_match_expected(expected_values, params_table, {38:  0.12})
-
     def _assert_fitted_positions_match_expected(self, expected_positions, params_table, rel_tolerance=None,
                                                 default_rel_tol=TestConstants.DEFAULT_RELATIVE_TOLERANCE):
         """

From 70a9e4a9ccbd14283b6d10c8dd000a933633c6da Mon Sep 17 00:00:00 2001
From: MialLewis <95620982+MialLewis@users.noreply.github.com>
Date: Tue, 2 May 2023 11:08:22 +0100
Subject: [PATCH 11/15] respond to pr review comments

---
 .../calibrate_vesuvio_analysis.py             | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
index 34d1933e..016593be 100644
--- a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
+++ b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
@@ -1,3 +1,4 @@
+<<<<<<< HEAD
 from mantid.kernel import StringArrayProperty, Direction, StringListValidator, FloatArrayBoundedValidator, StringMandatoryValidator,\
      IntBoundedValidator, FloatArrayProperty
 from mantid.api import FileProperty, FileAction, PythonAlgorithm, AlgorithmManager
@@ -5,6 +6,28 @@
      ConvertTableToMatrixWorkspace, ConjoinWorkspaces, Transpose, PlotPeakByLogValue, RenameWorkspace
 from calibration_scripts.calibrate_vesuvio_helper_functions import EVSGlobals, EVSMiscFunctions, InvalidDetectors
 
+||||||| parent of 1fa741e (respond to pr review comments)
+from mantid.kernel import StringArrayProperty, Direction, StringListValidator, IntArrayBoundedValidator, IntArrayProperty,\
+     FloatArrayBoundedValidator, FloatArrayMandatoryValidator, StringMandatoryValidator, IntBoundedValidator,\
+     FloatArrayProperty
+from mantid.api import FileProperty, FileAction, ITableWorkspaceProperty, PropertyMode, Progress, TextAxis, PythonAlgorithm,\
+     AlgorithmManager
+from mantid.simpleapi import CreateEmptyTableWorkspace, DeleteWorkspace, CropWorkspace, RebinToWorkspace, Divide,\
+     ReplaceSpecialValues, FindPeaks, GroupWorkspaces, mtd, Plus, LoadVesuvio, LoadRaw, ConvertToDistribution,\
+     FindPeakBackground, ExtractSingleSpectrum, SumSpectra, AppendSpectra, ConvertTableToMatrixWorkspace,\
+     ConjoinWorkspaces, Transpose, PlotPeakByLogValue, CloneWorkspace, MaskDetectors,\
+     ExtractUnmaskedSpectra, CreateWorkspace, RenameWorkspace
+from unpackaged.vesuvio_calibration.calibration_scripts.calibrate_vesuvio_helper_functions import EVSGlobals, EVSMiscFunctions,\
+     InvalidDetectors
+=======
+from mantid.kernel import StringArrayProperty, Direction, StringListValidator, IntArrayBoundedValidator, IntArrayProperty,\
+     FloatArrayBoundedValidator, StringMandatoryValidator, IntBoundedValidator,FloatArrayProperty
+from mantid.api import FileProperty, FileAction, PythonAlgorithm,AlgorithmManager
+from mantid.simpleapi import CreateEmptyTableWorkspace, DeleteWorkspace, ReplaceSpecialValues, GroupWorkspaces, mtd,\
+     ConvertTableToMatrixWorkspace, ConjoinWorkspaces, Transpose, PlotPeakByLogValue,RenameWorkspace
+from unpackaged.vesuvio_calibration.calibration_scripts.calibrate_vesuvio_helper_functions import EVSGlobals, EVSMiscFunctions,\
+     InvalidDetectors
+>>>>>>> 1fa741e (respond to pr review comments)
 
 import os
 import sys

From a5d8f04dc9e1773e8a7841788f5cb72530266db4 Mon Sep 17 00:00:00 2001
From: MialLewis <95620982+MialLewis@users.noreply.github.com>
Date: Tue, 2 May 2023 11:51:59 +0100
Subject: [PATCH 12/15] update paths to run from calibration root

---
 .../calibration_scripts/calibrate_vesuvio_analysis.py       | 2 +-
 .../tests/unit/test_calibrate_vesuvio_misc.py               | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
index 016593be..34d9518c 100644
--- a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
+++ b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
@@ -25,7 +25,7 @@
 from mantid.api import FileProperty, FileAction, PythonAlgorithm,AlgorithmManager
 from mantid.simpleapi import CreateEmptyTableWorkspace, DeleteWorkspace, ReplaceSpecialValues, GroupWorkspaces, mtd,\
      ConvertTableToMatrixWorkspace, ConjoinWorkspaces, Transpose, PlotPeakByLogValue,RenameWorkspace
-from unpackaged.vesuvio_calibration.calibration_scripts.calibrate_vesuvio_helper_functions import EVSGlobals, EVSMiscFunctions,\
+from calibration_scripts.calibrate_vesuvio_helper_functions import EVSGlobals, EVSMiscFunctions,\
      InvalidDetectors
 >>>>>>> 1fa741e (respond to pr review comments)
 
diff --git a/unpackaged/vesuvio_calibration/tests/unit/test_calibrate_vesuvio_misc.py b/unpackaged/vesuvio_calibration/tests/unit/test_calibrate_vesuvio_misc.py
index ec4fc6a4..6cdc1ca5 100644
--- a/unpackaged/vesuvio_calibration/tests/unit/test_calibrate_vesuvio_misc.py
+++ b/unpackaged/vesuvio_calibration/tests/unit/test_calibrate_vesuvio_misc.py
@@ -107,7 +107,7 @@ def test_get_all_detectors(self):
         invalid_detectors = InvalidDetectors([10, 20, 30, 150, 160, 170])
         self.assertEqual(invalid_detectors.get_all_invalid_detectors(), input_invalid_detectors)
 
-    @patch('unpackaged.vesuvio_calibration.calibration_scripts.calibrate_vesuvio_helper_functions'
+    @patch('calibration_scripts.calibrate_vesuvio_helper_functions'
            '.EVSMiscFunctions.read_fitting_result_table_column')
     def test_filter_peak_centres_for_invalid_detectors_front(self, mock_read_fitting_result):
         invalid_detectors = InvalidDetectors([10, 20, 30, 150, 160, 170])
@@ -117,7 +117,7 @@ def test_filter_peak_centres_for_invalid_detectors_front(self, mock_read_fitting
         out_peak_centres = invalid_detectors.filter_peak_centres_for_invalid_detectors([3, 134], peak_table)
         self.assertEqual(list(np.argwhere(np.isnan(out_peak_centres)).transpose()[0]), [7, 17, 27])
 
-    @patch('unpackaged.vesuvio_calibration.calibration_scripts.calibrate_vesuvio_helper_functions'
+    @patch('calibration_scripts.calibrate_vesuvio_helper_functions'
            '.EVSMiscFunctions.read_fitting_result_table_column')
     def test_filter_peak_centres_for_invalid_detectors_back(self, mock_read_fitting_result):
         invalid_detectors = InvalidDetectors([10, 20, 30, 150, 160, 170])
@@ -127,7 +127,7 @@ def test_filter_peak_centres_for_invalid_detectors_back(self, mock_read_fitting_
         out_peak_centres = invalid_detectors.filter_peak_centres_for_invalid_detectors([135, 198], peak_table)
         self.assertEqual(list(np.argwhere(np.isnan(out_peak_centres)).transpose()[0]), [15, 25, 35])
 
-    @patch('unpackaged.vesuvio_calibration.calibration_scripts.calibrate_vesuvio_helper_functions'
+    @patch('calibration_scripts.calibrate_vesuvio_helper_functions'
            '.EVSMiscFunctions.read_fitting_result_table_column')
     def test_filter_peak_centres_for_invalid_detectors_invalid_range(self, mock_read_fitting_result):
         invalid_detectors = InvalidDetectors([10, 20, 30, 150, 160, 170])

From cc58cbbdf1c60fce5654eda39f2bd871bf82182b Mon Sep 17 00:00:00 2001
From: MialLewis <95620982+MialLewis@users.noreply.github.com>
Date: Wed, 3 May 2023 13:21:50 +0100
Subject: [PATCH 13/15] resolve conflict in analysis imports

---
 .../calibrate_vesuvio_analysis.py             | 23 -------------------
 1 file changed, 23 deletions(-)

diff --git a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
index 34d9518c..f439f60f 100644
--- a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
+++ b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_analysis.py
@@ -1,25 +1,3 @@
-<<<<<<< HEAD
-from mantid.kernel import StringArrayProperty, Direction, StringListValidator, FloatArrayBoundedValidator, StringMandatoryValidator,\
-     IntBoundedValidator, FloatArrayProperty
-from mantid.api import FileProperty, FileAction, PythonAlgorithm, AlgorithmManager
-from mantid.simpleapi import CreateEmptyTableWorkspace, DeleteWorkspace, ReplaceSpecialValues, GroupWorkspaces, mtd,\
-     ConvertTableToMatrixWorkspace, ConjoinWorkspaces, Transpose, PlotPeakByLogValue, RenameWorkspace
-from calibration_scripts.calibrate_vesuvio_helper_functions import EVSGlobals, EVSMiscFunctions, InvalidDetectors
-
-||||||| parent of 1fa741e (respond to pr review comments)
-from mantid.kernel import StringArrayProperty, Direction, StringListValidator, IntArrayBoundedValidator, IntArrayProperty,\
-     FloatArrayBoundedValidator, FloatArrayMandatoryValidator, StringMandatoryValidator, IntBoundedValidator,\
-     FloatArrayProperty
-from mantid.api import FileProperty, FileAction, ITableWorkspaceProperty, PropertyMode, Progress, TextAxis, PythonAlgorithm,\
-     AlgorithmManager
-from mantid.simpleapi import CreateEmptyTableWorkspace, DeleteWorkspace, CropWorkspace, RebinToWorkspace, Divide,\
-     ReplaceSpecialValues, FindPeaks, GroupWorkspaces, mtd, Plus, LoadVesuvio, LoadRaw, ConvertToDistribution,\
-     FindPeakBackground, ExtractSingleSpectrum, SumSpectra, AppendSpectra, ConvertTableToMatrixWorkspace,\
-     ConjoinWorkspaces, Transpose, PlotPeakByLogValue, CloneWorkspace, MaskDetectors,\
-     ExtractUnmaskedSpectra, CreateWorkspace, RenameWorkspace
-from unpackaged.vesuvio_calibration.calibration_scripts.calibrate_vesuvio_helper_functions import EVSGlobals, EVSMiscFunctions,\
-     InvalidDetectors
-=======
 from mantid.kernel import StringArrayProperty, Direction, StringListValidator, IntArrayBoundedValidator, IntArrayProperty,\
      FloatArrayBoundedValidator, StringMandatoryValidator, IntBoundedValidator,FloatArrayProperty
 from mantid.api import FileProperty, FileAction, PythonAlgorithm,AlgorithmManager
@@ -27,7 +5,6 @@
      ConvertTableToMatrixWorkspace, ConjoinWorkspaces, Transpose, PlotPeakByLogValue,RenameWorkspace
 from calibration_scripts.calibrate_vesuvio_helper_functions import EVSGlobals, EVSMiscFunctions,\
      InvalidDetectors
->>>>>>> 1fa741e (respond to pr review comments)
 
 import os
 import sys

From aede6012482443b264927a68d19d3feb9ab18b41 Mon Sep 17 00:00:00 2001
From: MialLewis <95620982+MialLewis@users.noreply.github.com>
Date: Wed, 3 May 2023 15:32:02 +0100
Subject: [PATCH 14/15] do not identify if preset and blank

---
 .../calibrate_vesuvio_helper_functions.py     | 14 ++++--
 .../tests/unit/test_calibrate_vesuvio_misc.py | 44 +++++++++++++++++++
 2 files changed, 54 insertions(+), 4 deletions(-)

diff --git a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py
index f2ff879d..ae86398e 100644
--- a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py
+++ b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py
@@ -192,8 +192,14 @@ def generate_fit_function_header(function_type, error=False):
 class InvalidDetectors:
 
     def __init__(self, invalid_detector_list):
-        self._invalid_detectors_front = self._preset_invalid_detectors(invalid_detector_list, EVSGlobals.FRONTSCATTERING_RANGE)
-        self._invalid_detectors_back = self._preset_invalid_detectors(invalid_detector_list, EVSGlobals.BACKSCATTERING_RANGE)
+        if invalid_detector_list:
+            self._invalid_detectors_front = self._preset_invalid_detectors(invalid_detector_list, EVSGlobals.FRONTSCATTERING_RANGE)
+            self._invalid_detectors_back = self._preset_invalid_detectors(invalid_detector_list, EVSGlobals.BACKSCATTERING_RANGE)
+            self._detectors_preset = True
+        else:
+            self._invalid_detectors_front = np.array([])
+            self._invalid_detectors_back = np.array([])
+            self._detectors_preset = False
 
     def add_invalid_detectors(self, invalid_detector_list):
         """
@@ -248,13 +254,13 @@ def identify_and_set_invalid_detectors_from_range(self, detector_range, peak_tab
         peak_centres_errors = EVSMiscFunctions.read_fitting_result_table_column(peak_table, 'f1.LorentzPos_Err', detector_range)
 
         if detector_range == EVSGlobals.FRONTSCATTERING_RANGE:
-            if not self._invalid_detectors_front.any():
+            if not self._detectors_preset and not self._invalid_detectors_front.any():
                 self._invalid_detectors_front = self._identify_invalid_spectra(peak_table, peak_centres, peak_centres_errors,
                                                                                detector_range)
                 self._print_invalid_detectors(self._invalid_detectors_front, detector_range)
             return self._invalid_detectors_front
         elif detector_range == EVSGlobals.BACKSCATTERING_RANGE:
-            if not self._invalid_detectors_back.any():
+            if not self._detectors_preset and not self._invalid_detectors_back.any():
                 self._invalid_detectors_back = self._identify_invalid_spectra(peak_table, peak_centres, peak_centres_errors,
                                                                               detector_range)
                 self._print_invalid_detectors(self._invalid_detectors_back, detector_range)
diff --git a/unpackaged/vesuvio_calibration/tests/unit/test_calibrate_vesuvio_misc.py b/unpackaged/vesuvio_calibration/tests/unit/test_calibrate_vesuvio_misc.py
index 6cdc1ca5..fe45943c 100644
--- a/unpackaged/vesuvio_calibration/tests/unit/test_calibrate_vesuvio_misc.py
+++ b/unpackaged/vesuvio_calibration/tests/unit/test_calibrate_vesuvio_misc.py
@@ -86,16 +86,19 @@ def test_create_empty_invalid_detectors(self):
         invalid_detectors = InvalidDetectors([])
         self.assertEqual(invalid_detectors._invalid_detectors_back.tolist(), [])
         self.assertEqual(invalid_detectors._invalid_detectors_front.tolist(), [])
+        self.assertEqual(invalid_detectors._detectors_preset, False)
 
     def test_create_invalid_detectors_back(self):
         invalid_detectors = InvalidDetectors([10, 20, 30])
         self.assertEqual(invalid_detectors._invalid_detectors_back.tolist(), [[7], [17], [27]])
         self.assertEqual(invalid_detectors._invalid_detectors_front.tolist(), [])
+        self.assertEqual(invalid_detectors._detectors_preset, True)
 
     def test_create_invalid_detectors_front(self):
         invalid_detectors = InvalidDetectors([150, 160, 170])
         self.assertEqual(invalid_detectors._invalid_detectors_back.tolist(), [])
         self.assertEqual(invalid_detectors._invalid_detectors_front.tolist(), [[15], [25], [35]])
+        self.assertEqual(invalid_detectors._detectors_preset, True)
 
     def test_create_invalid_detectors(self):
         invalid_detectors = InvalidDetectors([10, 20, 30, 150, 160, 170])
@@ -155,6 +158,47 @@ def test_add_invalid_detectors(self):
         invalid_detectors.add_invalid_detectors([10, 20, 25, 30, 180, 190])
         self.assertEqual(invalid_detectors.get_all_invalid_detectors(), [10, 20, 25, 30, 150, 160, 170, 180, 190])
 
+    @patch('calibration_scripts.calibrate_vesuvio_helper_functions'
+           '.InvalidDetectors._identify_invalid_spectra')
+    @patch('calibration_scripts.calibrate_vesuvio_helper_functions'
+           '.EVSMiscFunctions.read_fitting_result_table_column')
+    def test_add_invalid_detectors_no_preset_identify_called_front(self, mock_read_fitting_result, mock_identify):
+        invalid_detectors = InvalidDetectors([])
+        invalid_detectors.add_invalid_detectors([180, 190])
+        self.assertEqual(invalid_detectors.get_all_invalid_detectors(), [180, 190])
+        self.assertEqual(invalid_detectors._detectors_preset, False)
+
+        peak_table = 'input_peak_table'
+        invalid_detectors.filter_peak_centres_for_invalid_detectors([3, 134], peak_table)
+        mock_identify.assert_called_once()
+
+    @patch('calibration_scripts.calibrate_vesuvio_helper_functions'
+           '.InvalidDetectors._identify_invalid_spectra')
+    @patch('calibration_scripts.calibrate_vesuvio_helper_functions'
+           '.EVSMiscFunctions.read_fitting_result_table_column')
+    def test_add_invalid_detectors_no_preset_identify_called_back(self, mock_read_fitting_result, mock_identify):
+        invalid_detectors = InvalidDetectors([])
+        invalid_detectors.add_invalid_detectors([31, 32])
+        self.assertEqual(invalid_detectors.get_all_invalid_detectors(), [31, 32])
+        self.assertEqual(invalid_detectors._detectors_preset, False)
+
+        peak_table = 'input_peak_table'
+        invalid_detectors.filter_peak_centres_for_invalid_detectors([135, 198], peak_table)
+        mock_identify.assert_called_once()
+
+    @patch('calibration_scripts.calibrate_vesuvio_helper_functions'
+           '.InvalidDetectors._identify_invalid_spectra')
+    @patch('calibration_scripts.calibrate_vesuvio_helper_functions'
+           '.EVSMiscFunctions.read_fitting_result_table_column')
+    def test_add_invalid_detectors_preset_identify_not_called(self, mock_read_fitting_result, mock_identify):
+        invalid_detectors = InvalidDetectors([180, 190])
+        self.assertEqual(invalid_detectors.get_all_invalid_detectors(), [180, 190])
+        self.assertEqual(invalid_detectors._detectors_preset, True)
+
+        peak_table = 'input_peak_table'
+        invalid_detectors.filter_peak_centres_for_invalid_detectors([3, 134], peak_table)
+        mock_identify.assert_not_called()
+
 
 if __name__ == '__main__':
     unittest.main()

From f2ee0d5762875956c7fc8f6410d1c841fc32ee26 Mon Sep 17 00:00:00 2001
From: MialLewis <95620982+MialLewis@users.noreply.github.com>
Date: Thu, 11 May 2023 11:22:32 +0100
Subject: [PATCH 15/15] repond to pr comments

---
 .../calibrate_vesuvio_helper_functions.py          | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py
index ae86398e..9964e22a 100644
--- a/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py
+++ b/unpackaged/vesuvio_calibration/calibration_scripts/calibrate_vesuvio_helper_functions.py
@@ -257,19 +257,25 @@ def identify_and_set_invalid_detectors_from_range(self, detector_range, peak_tab
             if not self._detectors_preset and not self._invalid_detectors_front.any():
                 self._invalid_detectors_front = self._identify_invalid_spectra(peak_table, peak_centres, peak_centres_errors,
                                                                                detector_range)
-                self._print_invalid_detectors(self._invalid_detectors_front, detector_range)
+                self._print_invalid_detectors(front_detector_range=True)
             return self._invalid_detectors_front
         elif detector_range == EVSGlobals.BACKSCATTERING_RANGE:
             if not self._detectors_preset and not self._invalid_detectors_back.any():
                 self._invalid_detectors_back = self._identify_invalid_spectra(peak_table, peak_centres, peak_centres_errors,
                                                                               detector_range)
-                self._print_invalid_detectors(self._invalid_detectors_back, detector_range)
+                self._print_invalid_detectors(front_detector_range=False)
             return self._invalid_detectors_back
         else:
             raise AttributeError("Spec list invalid - must represent either front or back detectors.")
 
-    @staticmethod
-    def _print_invalid_detectors(invalid_detectors, detector_range):
+    def _print_invalid_detectors(self, front_detector_range):
+        if front_detector_range:
+            invalid_detectors = self._invalid_detectors_front
+            detector_range = EVSGlobals.FRONTSCATTERING_RANGE
+        else:
+            invalid_detectors = self._invalid_detectors_back
+            detector_range = EVSGlobals.BACKSCATTERING_RANGE
+
         print(f'Invalid Spectra Index Found and Marked NAN: {invalid_detectors} from Spectra Index List:'
               f'{[x - EVSGlobals.DETECTOR_RANGE[0] for x in detector_range]}')