hw-mgmt: thermal: Fix recovery flow for special attention FAN

Fix FAN recovery flow not completing properly

In some cases, when the FAN recovery flow started, the FAN PWM was
immediately reset to its default value. Because of this, the recovery
process failed to complete.

The issue occurred when a FAN fault was triggered while the FAN PWM was
being adjusted downward in step increments.

Bug 4661922

Signed-off-by: Oleksandr Shamray <[email protected]>

Author: sholeksandr

usr/usr/bin/hw_management_thermal_control.py

@@ -3173,14 +3173,22 @@ def _is_attention_fan_insertion_fail(self):
     # ---------------------------------------------------------------------
     def _attention_fan_insertion_recovery(self):
         pwm = self.read_pwm(100)
+        if pwm <= self.fan_steady_state_pwm:
+            self.log.info("Recovery is not needed. Fan speed {}% is already <= steady state pwm: {}%".format(pwm, self.fan_steady_state_pwm), repeat=1)
+            return
+
         self.log.notice("Attention fan not started after insertion: Setting pwm to {}% from {}%".format(self.fan_steady_state_pwm, pwm), repeat=1)
+        if self.pwm_worker_timer and self.pwm_worker_timer.is_running():
+            self.pwm_worker_timer.stop()
         self._update_chassis_fan_speed(self.fan_steady_state_pwm, force=True)
         self.log.info("Waiting {}s for newly inserted fan to stabilize".format(self.fan_steady_state_delay))
         timeout = current_milli_time() + 1000 * self.fan_steady_state_delay
         while timeout > current_milli_time():
             self.exit.wait(1)
         self.log.info("Resuming normal operation: Setting pwm back to {}%".format(pwm))
         self._update_chassis_fan_speed(pwm, force=True)
+        if self.pwm_worker_timer:
+            self.pwm_worker_timer.start()
 
     # ----------------------------------------------------------------------
     def _update_psu_fan_speed(self, pwm):
@@ -3249,8 +3257,8 @@ def _set_pwm(self, pwm, reason="", force_reason=False):
             self.log.notice("PWM target changed from {} to PWM {} {}".format(self.pwm_target, pwm, reason))
             self._update_psu_fan_speed(pwm)
             self.pwm_target = pwm
-            if self.pwm_worker_timer:
-                self.pwm_worker_timer.start(True)
+            if self.pwm_worker_timer and not self.pwm_worker_timer.is_running():
+                self.pwm_worker_timer.start()
             else:
                 self.pwm = pwm
                 self._update_chassis_fan_speed(self.pwm)

usr/usr/bin/hw_management_thermal_control_2_5.py

@@ -3547,14 +3547,22 @@ def _is_attention_fan_insertion_fail(self):
     # ---------------------------------------------------------------------
     def _attention_fan_insertion_recovery(self):
         pwm = self.read_pwm(100)
+        if pwm <= self.fan_steady_state_pwm:
+            self.log.info("Recovery is not needed. Fan speed {}% is already <= steady state pwm: {}%".format(pwm, self.fan_steady_state_pwm), repeat=1)
+            return
+
         self.log.notice("Attention fan not started after insertion: Setting pwm to {}% from {}%".format(self.fan_steady_state_pwm, pwm), repeat=1)
+        if self.pwm_worker_timer and self.pwm_worker_timer.is_running():
+            self.pwm_worker_timer.stop()
         self._update_chassis_fan_speed(self.fan_steady_state_pwm, force=True)
         self.log.info("Waiting {}s for newly inserted fan to stabilize".format(self.fan_steady_state_delay))
         timeout = current_milli_time() + 1000 * self.fan_steady_state_delay
         while timeout > current_milli_time():
             self.exit.wait(1)
         self.log.info("Resuming normal operation: Setting pwm back to {}%".format(pwm))
         self._update_chassis_fan_speed(pwm, force=True)
+        if self.pwm_worker_timer:
+            self.pwm_worker_timer.start()
 
     # ----------------------------------------------------------------------
     def _update_psu_fan_speed(self, pwm):
@@ -3652,8 +3660,8 @@ def _set_pwm(self, pwm, reason="", force_reason=False):
             self.log.notice("PWM target changed from {} to PWM {} {}".format(self.pwm_target, pwm, reason))
             self._update_psu_fan_speed(pwm)
             self.pwm_target = pwm
-            if self.pwm_worker_timer:
-                self.pwm_worker_timer.start(True)
+            if self.pwm_worker_timer and not self.pwm_worker_timer.is_running():
+                self.pwm_worker_timer.start()
             else:
                 self.pwm = pwm
                 self._update_chassis_fan_speed(self.pwm)