initial commit

Author: majuid

.gitignore

@@ -0,0 +1,11 @@
+DeepNav_data/*
+DeepNav_results/*
+recycle/*
+*pycache*
+*.csv
+*.ulg
+*.pdf
+*.hdf5
+*.mp4
+*.json
+*.mat

DeepNav.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+"""
+Created on Mon 09 Nov 2020 | 5:25 PM
+
+@author: Ahmed Majuid
+
+Usage: Define the network architecture and training hyperparameters
+"""
+
+# non standard library
+import os
+import tensorflow as tf
+import training
+import utils
+import postprocessing
+from preprocessing.create_dataset import create_dataset
+
+# Session Parameters
+trial_number = 10
+
+session_mode = ["Fresh", "Resume", "Evaluate", "Override"]
+mode_id = 0
+gpu_name = ["/GPU:0", "/GPU:1", None]
+gpu_id = 0
+
+create_new_dataset = 1 # 0:No, 1:Yes
+
+# Network Hyperparameters
+batch_size = int(3 * 1024)
+learning_rate = 0.005
+dropout = 0.0
+epochs = 150
+initial_epoch = 0
+window_size = 50
+
+# Network Architecture
+model_architecture = [
+    tf.keras.layers.LSTM(100, return_sequences=True),
+    tf.keras.layers.LSTM(100, return_sequences=True),
+    tf.keras.layers.LSTM(100, return_sequences=True),
+    tf.keras.layers.LSTM(100, return_sequences=False),
+    tf.keras.layers.Dense(6)
+    ]
+
+n_features = 10
+n_labels = 6
+
+# Save the hyperparameters in a dictionary
+session_data = {"trial_number" : trial_number,
+                "session_mode" : session_mode[mode_id],
+                "gpu_name" : gpu_name[gpu_id],
+                "learning_rate" : learning_rate,
+                "window_size" : window_size,
+                "dropout" : dropout,
+                "batch_size" : batch_size,
+                "epochs" : epochs,
+                "initial_epoch" : initial_epoch,
+
+                "n_features" : n_features,
+                "n_labels" : n_labels,
+                }
+
+# create folders for the training outputs (weights, plots, loss history)
+trial_tree = utils.create_trial_tree(session_data["trial_number"], session_data["session_mode"])
+
+if create_new_dataset:
+    session_data["dataset_name"] = None
+else:
+    session_data["dataset_name"] = "T022_7logs_F11L10_W100_11Nov2020_1652"
+    
+# create windowed datasets from the flight csv files (or retrieve an old one from binary files)
+train_ds, val_dataset, train_flights_dict, val_flights_dict, signals_weights = create_dataset(session_data)
+
+# batch and shuffle
+train_dataset = train_ds.batch(batch_size).shuffle(buffer_size=1000)
+val_dataset = val_dataset.batch(batch_size).shuffle(buffer_size=1000)
+
+# convert signals weights to a tensor to be used by the loss function
+signals_weights_tensor = tf.constant(signals_weights, dtype=tf.float32)
+
+# start training
+model = training.start_training(session_data, model_architecture, train_dataset, val_dataset, \ 
+                                signals_weights_tensor, trial_tree)
+
+# for every flight, plot all states (truth vs predictions)
+postprocessing.evaluate_all_flights(model, train_flights_dict, val_flights_dict, \
+                                    trial_tree["trial_root_folder"], n_extreme_flights=10)
+
+# add the network configuration and performance to the summary csv
+postprocessing.summarize_session(trial_tree, model, session_data)

Readme.md

@@ -0,0 +1,32 @@
+# Readme
+
+## DeepNav Directory Structure
+
+DeepNav
+    |_ DeepNav_data : contains all data used for training
+        |_ ulg_files: log files as downloaded from the database (.ulg)
+        |_ flight_csvs: for every ulg file, there is a folder containing
+                        multiple csv files, one csv for each message
+                        at least there sould be
+                            -estimator_status
+                            -sensor_combined
+                            -vehicle_air_data
+                            -vehicle_magnetometer
+        |_ combined_csvs: multiple csvs combined into one csv per flight
+            |_ untrimmed
+                |_ original
+                |_ differenced: deltas using np.diff
+            |_ flight_names.csv: contains names of all flights
+                                in front of a name you can right the start and
+                                end minutes you wish to keep the data in between
+                                (trim outward) .. you can write "delete" to delete
+                                the flight 
+            |_ trimmed_csvs
+                |_ original
+                |_ differenced
+                    |_training
+                    |_validation
+            |_ deleted_csvs
+                |_ original
+                |_ differenced
+            

loss_plotter.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jun 12 21:48:47 2020
+
+@author: Ahmed Majuid
+"""
+
+import pandas as pd
+import matplotlib.pyplot as plt
+import numpy as np
+import os
+
+os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+# trials you want to plot
+trials = [11]
+legends = []
+
+for trial in trials:
+    
+    trial_log_file = os.path.join("DeepNav_results", "trial_" + str(trial).zfill(3), "model_history_log.csv")
+    # check if there is a log csv file for this trial and read it as a pandas frame
+    try:
+        df = pd.read_csv(trial_log_file)
+    except:
+        print("didn't find ", trial_log_file)
+        continue
+    
+    # extract the training and validation losses vectors from the data frame
+    df.head()
+    df = df[["loss", "val_loss"]]
+    
+    loss = df.loss
+    val_loss = df.val_loss
+    
+    # if you want to plot starting from a certain epoch (for zooming purposes)
+    zoom = 3
+    
+    epoch = np.linspace(1, len(loss)-1, len(loss)) - zoom
+    
+    # print total epochs, minimum loss and val_loss along with their epochs
+    print("trial:", trial, ", epochs:", val_loss.size, end=", ")
+    print("min_loss:", f'{np.min(loss):.4f}', "at epoch", np.argmin(loss), end=", ") 
+    print("min val_loss:", f'{np.min(val_loss):.4f}', "at epoch", np.argmin(val_loss),)
+
+
+    plt.plot(epoch[zoom:], loss[zoom:])
+    plt.plot(epoch[zoom:], val_loss[zoom:])
+    plt.grid(True)
+    
+    # add this trial labels to the legend
+    legends.append("trial " + str(trial).zfill(3) + " training loss")
+    legends.append("trial " + str(trial).zfill(3) + " validation loss")
+
+
+
+plt.legend(legends)
+plt.xlabel("epochs")
+plt.ylabel("Loss (MAE)")
+
+plt.savefig("losses.pdf")
+
+plt.show()