Improves Clustering and Mode tracking (#22)

- Fixes errors in clustering and mode tracking code.
  Refactors the code into smaller functions.  
- Adds new examples to showcase new capabilities
- Updates config format

---------
Co-authored-by: au650680 <au650680@uni.au.dk>

Author: au650680

config/production.json.template

@@ -1,29 +1,53 @@
 {
     "MQTT": {
-        "host": "test.mosquitto.org",
-        "port": 1883,
+        "host": "",
+        "port": ,
         "userId": "",
         "password": "",
-        "ClientID": "ReplaySubscriber",
+        "ClientID": "NOT_NEEDED",
         "QoS": 1,
         "TopicsToSubscribe": [
-            "cpsens/recorded/1/data",
-            "cpsens/recorded/1/metadata",
-            "cpsens/recorded/2/data",
-            "cpsens/recorded/+/data"
+          "cpsens/d8-3a-dd-37-d2-7e/3050-A-060_sn_106209/1/acc/raw/data",
+          "cpsens/d8-3a-dd-37-d2-7e/3050-A-060_sn_106209/1/acc/raw/metadata",
+          "cpsens/d8-3a-dd-37-d2-7e/3050-A-060_sn_106209/2/acc/raw/data",
+          "cpsens/d8-3a-dd-37-d2-7e/3050-A-060_sn_106209/3/acc/raw/data",
+          "cpsens/d8-3a-dd-37-d2-7e/3050-A-060_sn_106209/4/acc/raw/data"
         ]
       },
 
     "sysID": {
       "host": "",
-      "port": 1883,
+      "port": ,
       "userId": "",
       "password": "",
-      "ClientID": "sub.232.sds.213s",
-      "QoS": 1,
-      "TopicsToSubscribe": ["cpsens/d8-3a-dd-f5-92-48/cpsns_Simulator/1_2/oma_results"]
+      "ClientID": "NOT_NEEDED",
+      "QoS": 2,
+      "TopicsToSubscribe": ["cpsens/d8-3a-dd-37-d2-7e/3050-A-060_sn_106209/1/acc/sysid/data"]
+    },
+
+    "mode_cluster": {
+      "host": "",
+      "port": ,
+      "userId": "",
+        "password": "",
+      "ClientID": "NOT_NEEDED",
+      "QoS": 2,
+      "TopicsToSubscribe": ["cpsens/d8-3a-dd-37-d2-7e/3050-A-060_sn_106209/1/acc/mode_cluster/data"]
+    },
+
+    "model_update": {
+      "host": "",
+      "port": ,
+      "userId": "",
+        "password": "",
+      "ClientID": "NOT_NEEDED",
+      "QoS": 2,
+      "TopicsToSubscribe": ["cpsens/d8-3a-dd-37-d2-7e/3050-A-060_sn_106209/1/acc/model_update/data"]
     }
 }
 
 
 
+
+
+

pyproject.toml

@@ -16,7 +16,7 @@ paho-mqtt = "^2.1.0"
 numpy = "^2.2.5"
 click ="^8.1.8"
 pyOMA-2 = "1.0.0"
-yafem = { path = "src/methods/packages/yafem-0.2.6-py3-none-any.whl" }
+yafem = "1.0.0"
 
 
 [tool.poetry.group.dev.dependencies]

src/data/accel/hbk/accelerometer.py

@@ -86,7 +86,7 @@ def process_message(self, msg: mqtt.MQTTMessage) -> None:
                     if not oldest_deque:  # Remove the key/deque from the map if it's empty
                         del self.data_map[oldest_key]
                     total_samples = sum(len(dq) for dq in self.data_map.values())
-            print(f" Channel: {self.topic}  Key: {samples_from_daq_start}, Samples: {num_samples}")
+            #print(f" Channel: {self.topic}  Key: {samples_from_daq_start}, Samples: {num_samples}")
 
         except Exception as e:
             print(f"Error processing message: {e}")

src/data/accel/hbk/aligner.py

@@ -130,7 +130,7 @@ def _extract_aligned_block(self, group: List[int], batch_size: int,
             ch.clear_used_data(group[0], requested_samples)
 
         aligned_array = np.array(aligned_data, dtype=np.float32)
-        print(f"Aligned shape: {aligned_array.shape}")
+        print(f"\nAligned shape: {aligned_array.shape}")
         return aligned_array, utc_time
 
 

src/examples/README.md

@@ -62,13 +62,18 @@ To run the examples with the default config, use:
 ```bash
 python .\src\examples\example.py accelerometers
 python .\src\examples\example.py align-readings
-python .\src\examples\example.py oma-and-print
-python .\src\examples\example.py oma-and-plot
-python .\src\examples\example.py oma-and-publish
-python .\src\examples\example.py mode-tracking-with-local-sysid
-python .\src\examples\example.py mode-tracking-with-remote-sysid
+python .\src\examples\example.py sysid-and-print
+python .\src\examples\example.py sysid-and-plot
+python .\src\examples\example.py sysid-and-publish
+python .\src\examples\example.py live-sysid-publish
+python .\src\examples\example.py clustering-with-local-sysid
+python .\src\examples\example.py clustering-with-remote-sysid
+python .\src\examples\example.py live-clustering-with-remote-sysid
+python .\src\examples\example.py mode-tracking-remote-sysid
+python .\src\examples\example.py mode_tracking-with-local-sysid
+python .\src\examples\example.py live-mode-tracking-remote-sysid
 python .\src\examples\example.py model-update-local-sysid
-python .\src\examples\example.py model-update-remote-sysid
+python .\src\examples\example.py ilve-model-update-remote-sysid
 
 ```
 

src/examples/example.py

@@ -2,16 +2,23 @@
 import click
 from examples.acceleration_readings import read_accelerometers
 from examples.aligning_readings import align_acceleration_readings
-from examples.run_pyoma import (
-    run_oma_and_plot,
-    run_oma_and_publish,
-    run_oma_and_print,
+from examples.run_sysid import (
+    run_sysid_and_plot,
+    run_sysid_and_publish,
+    run_sysid_and_print,
+    live_sysid_and_publish
 )
-from examples.mode_tracking import (
+from examples.run_mode_clustering import (
+    run_mode_clustering_with_local_sysid,
+    run_mode_clustering_with_remote_sysid,
+    run_live_mode_clustering_with_remote_sysid
+)
+from examples.run_mode_tracking import (
     run_mode_tracking_with_local_sysid,
     run_mode_tracking_with_remote_sysid,
+    run_live_mode_tracking_with_remote_sysid
 )
-from examples.updating_parameters import (
+from examples.run_model_update import (
     run_model_update_local_sysid, 
     run_model_update_remote_sysid
 )
@@ -34,22 +41,40 @@ def accelerometers(ctx):
 def align_readings(ctx):
     align_acceleration_readings(ctx.obj["CONFIG"])
 
+@cli.command()
+@click.pass_context
+def sysid_and_publish(ctx):
+    run_sysid_and_publish(ctx.obj["CONFIG"])
 
 @cli.command()
 @click.pass_context
-def oma_and_publish(ctx):
-    run_oma_and_publish(ctx.obj["CONFIG"])
+def live_sysid_publish(ctx):
+    live_sysid_and_publish(ctx.obj["CONFIG"])
 
 @cli.command()
 @click.pass_context
-def oma_and_plot(ctx):
-    run_oma_and_plot(ctx.obj["CONFIG"])
+def sysid_and_plot(ctx):
+    run_sysid_and_plot(ctx.obj["CONFIG"])
 
 @cli.command()
 @click.pass_context
-def oma_and_print(ctx):
-    run_oma_and_print(ctx.obj["CONFIG"])
+def sysid_and_print(ctx):
+    run_sysid_and_print(ctx.obj["CONFIG"])
 
+@cli.command()
+@click.pass_context
+def clustering_with_local_sysid(ctx):
+    run_mode_clustering_with_local_sysid(ctx.obj["CONFIG"])
+
+@cli.command()
+@click.pass_context
+def clustering_with_remote_sysid(ctx):
+    run_mode_clustering_with_remote_sysid(ctx.obj["CONFIG"])
+
+@cli.command()
+@click.pass_context
+def live_clustering_with_remote_sysid(ctx):
+    run_live_mode_clustering_with_remote_sysid(ctx.obj["CONFIG"])
 
 @cli.command()
 @click.pass_context
@@ -61,6 +86,10 @@ def mode_tracking_with_local_sysid(ctx):
 def mode_tracking_with_remote_sysid(ctx):
     run_mode_tracking_with_remote_sysid(ctx.obj["CONFIG"])
 
+@cli.command()
+@click.pass_context
+def live_mode_tracking_with_remote_sysid(ctx):
+    run_live_mode_tracking_with_remote_sysid(ctx.obj["CONFIG"])
 
 @cli.command()
 @click.pass_context
@@ -69,7 +98,7 @@ def model_update_local_sysid(ctx):
 
 @cli.command()
 @click.pass_context
-def model_update_remote_sysid(ctx):
+def live_model_update_remote_sysid(ctx):
     run_model_update_remote_sysid(ctx.obj["CONFIG"])
 
 if __name__ == "__main__":

src/examples/mode_tracking.py

@@ -0,0 +1,53 @@
+import sys
+import time
+import matplotlib.pyplot as plt
+from data.comm.mqtt import load_config
+from data.accel.hbk.aligner import Aligner
+from methods import sysid as sysID
+from methods import mode_clustering as MC
+from methods.constants import PARAMS
+from functions.plot_clusters import plot_clusters
+
+# pylint: disable=R0914
+def run_mode_clustering_with_local_sysid(config_path):
+    number_of_minutes = 1
+    config = load_config(config_path)
+    mqtt_config = config["MQTT"]
+
+    # Setting up the client and extracting Fs
+    data_client, fs = sysID.setup_client(mqtt_config)
+
+    # Setting up the aligner
+    data_topic_indexes = [0, 2, 3, 4]
+    selected_topics = [mqtt_config["TopicsToSubscribe"][i] for i in data_topic_indexes]
+    aligner = Aligner(data_client, topics=selected_topics)
+
+    aligner_time = None
+    t1 = time.time()
+    while aligner_time is None:
+        time.sleep(0.1)
+        t2 = time.time()
+        t_text = f"Waiting for data for {round(t2-t1,1)} seconds"
+        print(t_text,end="\r")
+        sysid_output, aligner_time = sysID.get_sysid_results(number_of_minutes, aligner, fs)
+    data_client.disconnect()
+
+    # Mode Tracks
+    dictionary_of_clusters, median_frequencies = MC.cluster_sysid(
+        sysid_output,PARAMS)
+
+    # Print frequencies
+    print("\nMedian frequencies:", median_frequencies)
+
+    fig_ax = plot_clusters(dictionary_of_clusters, sysid_output, PARAMS, fig_ax = None)
+    plt.show(block=True)
+    sys.stdout.flush()
+
+def run_mode_clustering_with_remote_sysid(config_path):
+    sysid_output, dictionary_of_clusters, meadian_frequencies = MC.subscribe_and_cluster(config_path,PARAMS)
+    fig_ax = plot_clusters(dictionary_of_clusters, sysid_output, PARAMS, fig_ax = None)
+    plt.show(block=True)
+    sys.stdout.flush()
+
+def run_live_mode_clustering_with_remote_sysid(config_path):
+    MC.live_mode_clustering(config_path,topic_index=0,plot=[1,1])

src/examples/run_mode_clustering.py

@@ -0,0 +1,57 @@
+import sys
+import time
+import matplotlib.pyplot as plt
+from data.comm.mqtt import load_config
+from data.accel.hbk.aligner import Aligner
+from methods import sysid as sysID
+from methods import mode_clustering as MC
+from methods import mode_tracking as MT
+from methods.constants import PARAMS
+from functions.plot_mode_tracking import plot_tracked_modes
+
+# pylint: disable=R0914
+def run_mode_tracking_with_local_sysid(config_path):
+    number_of_minutes = 1
+    config = load_config(config_path)
+    mqtt_config = config["MQTT"]
+
+    # Setting up the client and extracting Fs
+    data_client, fs = sysID.setup_client(mqtt_config)
+
+    # Setting up the aligner
+    data_topic_indexes = [0, 2, 3, 4]
+    selected_topics = [mqtt_config["TopicsToSubscribe"][i] for i in data_topic_indexes]
+    aligner = Aligner(data_client, topics=selected_topics)
+
+    aligner_time = None
+    t1 = time.time()
+    while aligner_time is None:
+        time.sleep(0.1)
+        t2 = time.time()
+        t_text = f"Waiting for data for {round(t2-t1,1)} seconds"
+        print(t_text,end="\r")
+        sysid_output, aligner_time = sysID.get_sysid_results(number_of_minutes, aligner, fs)
+    data_client.disconnect()
+
+    # Mode Tracks
+    dictionary_of_clusters, median_frequencies = MC.cluster_sysid(
+        sysid_output,PARAMS)
+
+    # Print frequencies
+    print("\nMedian frequencies:", median_frequencies)
+
+    tracked_clusters = {}
+    tracked_clusters = MT.track_clusters(dictionary_of_clusters,tracked_clusters,PARAMS)
+
+    fig_ax = plot_tracked_modes(tracked_clusters, PARAMS, fig_ax = None, x_length = None)
+    plt.show(block=True)
+    sys.stdout.flush()
+
+def run_mode_tracking_with_remote_sysid(config_path):
+    sysid_output, clusters, tracked_clusters = MT.subscribe_and_track_clusters(config_path)
+    fig_ax = plot_tracked_modes(tracked_clusters, PARAMS, fig_ax = None, x_length = None)
+    plt.show(block=True)
+    sys.stdout.flush()
+
+def run_live_mode_tracking_with_remote_sysid(config_path):
+    MT.live_mode_tracking(config_path,plot=[1,1])