multimodal input code modifications for gym pybullet drones data

Author: makramchahine

.gitignore

@@ -0,0 +1,72 @@
+*.pyc
+
+save/
+log/
+
+tags
+
+# Ignore PyCharm idea files
+.idea
+__pycache__
+
+# caches
+*.h5
+
+# Data
+data/*
+**/model_checkpoints/*
+processed-data/*
+training-histories/*
+model-piloted-runs/*
+console-log/*
+junk/*
+history/*
+histories/*
+logs/*
+airsim/*
+visualbackprop/*
+setup_path.py
+*~
+*.db
+*.tf
+*spec.txt
+cached_data
+
+# pyenv
+.python-version
+
+# keras
+.cache/*
+.keras/*
+
+# vim
+*.swp
+
+aaron*
+*_logs
+tuner_tests/
+*.vscode
+
+# visual backprop results
+*.png
+*.mp4
+
+# tuning results
+*.err
+*.log*
+*.out
+*.pkl
+**/*train_results.json
+
+# dataset jsons
+#dataset_jsons/
+
+# analysis results
+**/lipschitz_out/
+**/ssim_out/
+**/perturb_out/
+**/grid_out/
+**/loss_out/
+
+# shap rep
+shap/

README.md

@@ -8,6 +8,7 @@
 from tensorflow.python.keras import Model
 from tensorflow.python.keras.layers import Conv2D
 from tensorflow.python.keras.models import Functional
+import numpy as np
 
 from utils.vis_utils import image_grid
 from utils.model_utils import load_model_from_weights, load_model_no_params, ModelParams
@@ -17,8 +18,14 @@ def compute_gradcam(img: Union[Tensor, ndarray], grad_model: Functional, hiddens
                     pred_index: Optional[Sequence[Tensor]] = None):
     heatmaps, hiddens = _compute_gradcam(img=img, grad_model=grad_model, hiddens=hiddens, pred_index=pred_index)
     avg_heat = tf.math.add_n(heatmaps)
+
+    map_min = np.min(avg_heat)
+    map_max = np.max(avg_heat)
+    avg_heat = (avg_heat - map_min) / (map_max - map_min + 1e-6)
+
     avg_heat = tf.expand_dims(avg_heat, axis=-1)
-    return avg_heat, hiddens
+
+    return avg_heat, hiddens, avg_heat
 
 
 def compute_gradcam_tile(img: Union[Tensor, ndarray], grad_model: Functional, hiddens: Sequence[Tensor],
@@ -47,6 +54,7 @@ def _compute_gradcam(img: Union[Tensor, ndarray], grad_model: Functional, hidden
     # Then, we compute the gradient of the top predicted class for our input image
     # with respect to the activations of the last conv layer
     with tf.GradientTape() as tape:
+        img = tf.convert_to_tensor(img, dtype=tf.float32)
         out = grad_model([img, *hiddens])
         last_conv_layer_output = out[0]
         preds = out[1]
@@ -59,7 +67,7 @@ def _compute_gradcam(img: Union[Tensor, ndarray], grad_model: Functional, hidden
     grads = tape.jacobian(preds, last_conv_layer_output)[0]
     last_conv_layer_output = last_conv_layer_output[0]
     for pred in pred_index:
-        # This is the gradient of the output neuron (top pred1icted or chosen)
+        # This is the gradient of the output neuron (top predicted or chosen)
         # with regard to the output feature map of the last conv layer
         grad = grads[pred]
 
@@ -95,5 +103,6 @@ def get_last_conv(model_path: str, model_params: Optional[ModelParams] = None) -
     # First, we create a model that maps the input image to the activations
     # of the last conv layer as well as the output predictions
     return tf.keras.models.Model(
-        [vis_model.inputs], [conv_layers[-1].output, *vis_model.output]
+        inputs=[vis_model.inputs],
+        outputs=[conv_layers[-1].output, *vis_model.output]
     )

analysis/grad_cam.py

@@ -10,7 +10,8 @@
 import seaborn as sns
 
 # noinspection PyArgumentList
-from utils.graph_utils import MARKERS
+# from utils.graph_utils import MARKERS
+MARKERS = ['o', 'v', '^', '<', '>', 's', 'p', '*', 'h', 'H', 'D', 'd']
 
 
 # noinspection PyArgumentList
@@ -42,6 +43,8 @@ def graph_loss(train_losses: Optional[Dict[str, Sequence[Tuple[int, float]]]] =
     plt.xlabel("Epoch")
     plt.ylabel("Loss")
     plt.legend(loc="upper right")
+    # make y axis logaritmic:
+    plt.yscale('log')
 
     plt.autoscale(enable=True, axis='x', tight=True)
 
@@ -81,6 +84,7 @@ def get_losses_from_checkpoints(checkpoint_paths: Sequence[str], save_dir: str,
     all_train_losses = {}
     all_val_losses = {}
     for i, checkpoint_path in enumerate(checkpoint_paths):
+        print(checkpoint_path)
         match_reg = re.compile(".*(\d\d\d\d:\d\d:\d\d:\d\d:\d\d:\d\d)")
         date_str = match_reg.search(checkpoint_path).group(1)
 

analysis/loss_graph.py

@@ -2,6 +2,8 @@
 
 import numpy as np
 import tensorflow as tf
+physical_devices = tf.config.list_physical_devices('GPU')
+tf.config.experimental.set_memory_growth(physical_devices[0], True)
 from numpy import ndarray
 from tensorflow import keras, Tensor
 from tensorflow.keras.layers import Conv2D
@@ -32,7 +34,6 @@ def compute_visualbackprop(img: Union[Tensor, ndarray],
             if isinstance(layer, Conv2D):
                 kernels.append(layer.kernel_size)
                 strides.append(layer.strides)
-
     activations = activation_model.predict(img)
     average_layer_maps = []
     for layer_activation in activations:  # Only the convolutional layers

analysis/visual_backprop.py

@@ -0,0 +1,3 @@
+Folder for storing legacy scripts that are no longer used or are deprecated. If a script in this directory is useful, move it outside.
+
+Scripts used for simulated tasks are also in this folder for now

archive/README.md

@@ -0,0 +1,140 @@
+import pickle
+import os
+import seaborn as sns
+import matplotlib.pyplot as plt
+import argparse
+import sys
+import numpy as np
+import pandas as pd
+
+REVISIONS = (5,6)
+
+sns.set_theme(style="darkgrid")
+parser = argparse.ArgumentParser(description='Plot training history of deepdrone project')
+parser.add_argument('--history_dir', type=str, default="history", help='The path to the training histories')
+args = parser.parse_args()
+
+enviromentNames = set()
+taskNames       = set()
+modelNames      = set()
+
+histories = dict()
+for domain in os.listdir(args.history_dir):
+
+    task = domain[:domain.index('-')]
+    enviroment = domain[domain.index('-')+1:]
+
+    if enviroment not in histories:
+        histories[enviroment] = dict()
+        enviromentNames.add(enviroment)
+
+    if task not in histories[enviroment]:
+        histories[enviroment][task] = dict()
+        taskNames.add(task)
+
+
+    for historyFile in os.listdir(args.history_dir + '/' + domain ):
+
+        if any([(("rev-" + str(r)) in historyFile) for r in REVISIONS]):
+            continue
+
+        # map each model type to its training histories
+        model = historyFile[:historyFile.index('-')]
+        if model not in histories[enviroment][task]:
+            histories[enviroment][task][model] = []
+            modelNames.add(model)
+
+        with open(args.history_dir + '/' + domain + '/' + historyFile, 'rb') as fp:
+            print(args.history_dir + '/' + domain + '/' + historyFile)
+            histories[enviroment][task][model].append(pickle.load(fp))
+
+enviromentNames = sorted(list(enviromentNames))
+taskNames       = sorted(list(taskNames))
+modelNames      = sorted(list(modelNames))
+
+
+for enviroment in enviromentNames:
+    for i, task in enumerate(taskNames):
+        validationFrame = []
+        for j, model in enumerate(modelNames):
+
+            try:
+                loss = [h["loss"] for h in histories[enviroment][task][model]]
+                validationLoss = [h["val_loss"] for h in histories[enviroment][task][model]]
+            except KeyError:
+                print(enviroment, task, model, "data missing")
+                print("")
+                continue
+
+            losses = np.array(loss)
+            validations = np.array(validationLoss)
+
+            for epoche, replicationSet in enumerate(validations.T):
+                for v in replicationSet:
+                    validationFrame.append([epoche, model, v])
+
+
+
+        validationFrame = pd.DataFrame(validationFrame, columns=["epoche", "model", "loss"])
+
+        sns.lineplot(data=validationFrame, x="epoche", y="loss", hue="model")
+        plt.title(enviroment + " " + task)
+        plt.show()
+
+
+
+# for enviroment in enviromentNames:
+#     fig, axes = plt.subplots(len(taskNames), len(modelNames))
+#     axes = np.reshape(axes, (len(taskNames), len(modelNames)))
+#     for i, task in enumerate(taskNames):
+#         for j, model in enumerate(modelNames):
+#             
+#             if j == 0:
+#                 axes[i, 0].set_ylabel(task)
+# 
+#             if i == 0:
+#                 axes[0, j].set_title(model)
+# 
+#             try:
+#                 loss = [h["loss"] for h in histories[enviroment][task][model]]
+#                 validationLoss = [h["val_loss"] for h in histories[enviroment][task][model]]
+#             except KeyError:
+#                 continue
+# 
+#             losses = np.array(loss)
+#             validations = np.array(validationLoss)
+# 
+#             lossStdDev = np.std(losses, axis=0)
+#             validationStdDev = np.std(validations, axis=0)
+# 
+#             lossMean = np.mean(losses, axis=0)
+#             validationMean = np.mean(validations, axis=0)
+# 
+#             print(enviroment, task, model)
+#             minIdx = np.argmin(validationMean)
+#             print(f"{validationMean[minIdx]:.3} +/- {validationStdDev[minIdx]:.3}")
+# 
+#             # for l in losses:
+#             #     axes[i, j].plot(range(len(l)), l, "r", label="losses")
+#             
+#             for v in validations:
+#                 logV = [np.log(1 + 100000000 * np.abs(v_i)) for v_i in v]
+#                 axes[i, j].plot(range(len(v)), logV, "b", label="validations")
+# 
+#             # axes[i, j].errorbar(range(len(lossMean)), lossMean, yerr=lossStdDev, label="Training Loss")
+#             # axes[i, j].errorbar(range(len(validationMean)), validationMean, yerr=validationStdDev, label="Validation Loss")
+#             # axes[i, j].legend()
+# 
+#             # print(enviroment, task, model)
+#             # print(loss.shape)
+#             # print(loss)
+# 
+#             # print(lossStdDev.shape)
+#             # print(lossStdDev)
+# 
+#             # sys.exit()
+# 
+#         print("")
+# 
+#     fig.suptitle(f"{enviroment} enviroment learning curves")
+#     plt.show()

archive/analysis.py

@@ -0,0 +1,19 @@
+import os
+import shutil
+import numpy as np
+
+DATASET_PATH = 'C:\\Users\\MIT Driverless\\Documents\\AirSim\\following-neighborhood-parsed'
+TRUNCATED_PATH = DATASET_PATH + '-truncated'
+
+if not os.path.exists(TRUNCATED_PATH):
+    os.makedirs(TRUNCATED_PATH)
+
+for dataset in os.listdir(DATASET_PATH):
+    print(dataset)
+    data_files = set(os.listdir(DATASET_PATH + '\\' + dataset))
+    for f in data_files:
+        data = np.load(DATASET_PATH + '\\' + dataset + '\\' + f)
+        uint8_data = (255*data).astype(np.uint8) 
+        if not os.path.exists(TRUNCATED_PATH + '\\' + dataset):
+            os.makedirs(TRUNCATED_PATH + '\\' + dataset)
+        np.save(TRUNCATED_PATH + '\\' + dataset + '\\' + f, uint8_data)

archive/compress-data.py

@@ -0,0 +1,45 @@
+import pickle
+import os
+import matplotlib.pyplot as plt
+import argparse
+import sys
+
+import numpy as np
+
+parser = argparse.ArgumentParser(description='Plot training history of deepdrone project')
+parser.add_argument('--history_dir', type=str, default="history", help='The path to the training histories')
+args = parser.parse_args()
+
+enviromentNames = set()
+taskNames       = set()
+modelNames      = set()
+
+histories = dict()
+for domain in os.listdir(args.history_dir):
+
+    task = domain[:domain.index('-')]
+    enviroment = domain[domain.index('-')+1:]
+
+    if enviroment not in histories:
+        histories[enviroment] = dict()
+        enviromentNames.add(enviroment)
+
+    if task not in histories[enviroment]:
+        histories[enviroment][task] = dict()
+        taskNames.add(task)
+
+
+    for i, historyFile in enumerate(os.listdir(args.history_dir + '/' + domain)):
+
+        # map each model type to its training histories
+        model = historyFile[:historyFile.index('-')]
+        if model not in histories[enviroment][task]:
+            histories[enviroment][task][model] = []
+            modelNames.add(model)
+
+        with open(args.history_dir + '/' + domain + '/' + historyFile, 'rb') as fp:
+            history = pickle.load(fp)
+            histories[enviroment][task][model].append(history)
+
+            np.savetxt(f"csv/{enviroment}-{task}-{model}-training-loss-{len(histories[enviroment][task][model])}", history["loss"], delimiter=",")
+            np.savetxt(f"csv/{enviroment}-{task}-{model}-validation-loss-{len(histories[enviroment][task][model])}", history["val_loss"], delimiter=",")