Add PPO evaluation

Closes: #75

examples/gymfc_nf/policies/__init__.py

@@ -1,3 +1,4 @@
 
 from gymfc_nf.policies.pidpolicy import PidPolicy
-__all__ = ['PidPolicy']
+from gymfc_nf.policies.baselinespolicy import PpoBaselinesPolicy
+__all__ = ['PidPolicy', 'PpoBaselinesPolicy']

examples/gymfc_nf/policies/baselinespolicy.py

@@ -0,0 +1,14 @@
+import numpy as np
+import tensorflow as tf
+from .policy import Policy
+class PpoBaselinesPolicy(Policy):
+ def __init__(self, sess):
+ graph = tf.get_default_graph()
+ self.x = graph.get_tensor_by_name('pi/ob:0') 
+ self.y = graph.get_tensor_by_name('pi/pol/final/BiasAdd:0')
+ self.sess = sess
+
+ def action(self, state, sim_time=0, desired=np.zeros(3), actual=np.zeros(3) ):
+
+ y_out = self.sess.run(self.y, feed_dict={self.x:[state] })
+ return y_out[0]

examples/gymfc_nf/utils/monitor.py

@@ -0,0 +1,63 @@
+import tensorflow as tf
+import os.path
+import time
+
+
+class CheckpointMonitor:
+ """Helper class to monitor the Tensorflow checkpoints and call a callback
+ when a new checkpoint has been created."""
+
+ def __init__(self, checkpoint_dir, callback):
+ """
+ Args:
+ checkpoint_dir: Directory to monitor where new checkpoint
+ directories will be created
+ callback: A callback for when a new checkpoint is created.
+ """
+ self.checkpoint_dir = checkpoint_dir
+ self.callback = callback
+ # Track which checkpoints have already been called. 
+ self.processed = []
+
+ self.watching = True
+
+ def _check_new_checkpoint(self):
+ """Update the queue with newly found checkpoints.
+
+ When a checkpoint directory is created a 'checkpoint' file is created
+ containing a list of all the checkpoints. We can monitor this file to
+ determine when new checkpoints have been created.
+ """
+ # TODO (wfk) check if there is a way to get a callback when a file has
+ # changed. 
+
+ ckpt = tf.train.get_checkpoint_state(self.checkpoint_dir)
+ for path in ckpt.all_model_checkpoint_paths:
+ checkpoint_filename = os.path.split(path)[-1]
+ if tf.train.checkpoint_exists(path):
+ # Make sure there is a checkpoint meta file before allowing it
+ # to be processed
+ meta_file = path + ".meta"
+ if os.path.isfile(meta_file):
+ if (checkpoint_filename not in self.processed):
+ self.callback(checkpoint_filename)
+ self.processed.append(checkpoint_filename)
+ else:
+ print ("Meta file {} doesn't exist.".format(meta_file))
+
+ def start(self):
+
+ # Sit and wait until the checkpoint directory is created, otherwise we
+ # can't monitor it. If it never gets created this could be an indicator
+ # something is wrong with the trainer.
+ c=0
+ while not os.path.isdir(self.checkpoint_dir):
+ print("[WARN {}] Directory {} doesn't exist yet, waiting until "
+ "created...".format(c, self.checkpoint_dir))
+ time.sleep(30)
+ c+=1
+
+ while self.watching:
+ self._check_new_checkpoint()
+ time.sleep(10)
+

examples/ppo_baselines_evaluate.py

@@ -0,0 +1,120 @@
+import argparse
+from pathlib import Path
+import os.path
+import numpy as np
+import tensorflow as tf
+import gym
+from gymfc_nf.envs import *
+from gymfc_nf.utils.monitor import CheckpointMonitor
+from gymfc_nf.policies import PpoBaselinesPolicy
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser("Evaluate OpenAI Baseline PPO1 checkpoints.")
+ parser.add_argument('ckpt_dir', help="Directory where checkpoints are saved. ")
+ parser.add_argument('--twin', default="./gymfc_nf/twins/nf1/model.sdf",
+ help="File path of the aircraft digitial twin/model SDF.")
+ parser.add_argument('--eval-dir', 
+ help="Directory where evaluation logs are saved.")
+ parser.add_argument('--gym-id', default="gymfc_nf-step-v1")
+ parser.add_argument('--num-trials', type=int, default=1)
+ # Provide a seed so the same setpoint will be created. Useful for debugging
+ parser.add_argument('--seed', help='RNG seed', type=int, default=-1)
+
+ args = parser.parse_args()
+
+ seed = np.random.randint(0, 1e6) if args.seed < 0 else args.seed
+ gym_id = args.gym_id
+ ckpt_dir = args.ckpt_dir
+ model_dir = Path(ckpt_dir).parent 
+ eval_dir = args.eval_dir if args.eval_dir else os.path.join(model_dir,
+ "evaluations")
+ num_trials = args.num_trials
+ print ("Saving evaluations to {}".format(eval_dir))
+
+ env = gym.make(gym_id)
+ env.seed(seed)
+ env.set_aircraft_model(args.twin)
+
+ log_header = ""
+ def make_header(ob_size):
+ """Make the log header.
+
+ This needs to be done dynamically because the observations which are 
+ used as input to the NN may differ.
+ """
+ entries = []
+ entries.append("t")
+ for i in range(ob_size):
+ entries.append("ob{}".format(i))
+ for i in range(4):
+ entries.append("ac{}".format(i))
+ for i in range(4):
+ entries.append("y{}".format(i))
+ entries.append("p") # roll rate
+ entries.append("q") # pitch rate
+ entries.append("r") # yaw rate
+ entries.append("p-sp") # roll rate setpoint
+ entries.append("q-sp") # pitch rate setpoint
+ entries.append("r-sp") # yaw rate setpoint
+ for i in range(4):
+ entries.append("w{}".format(i)) # ESC rpms
+ entries.append("reward")
+
+ log_header = ",".join(entries)
+
+ def callback(checkpoint):
+ print ("Callback ", checkpoint)
+
+ ckpt_eval_dir = os.path.join(eval_dir, checkpoint)
+ Path(ckpt_eval_dir).mkdir(parents=True, exist_ok=True)
+
+ # TODO (wfk) I'm pretty sure this just takes the last checkpoint 
+ # written defined by 'model_checkpoint_path' in the checkpoint file
+ # should look at how to specify the exact one. 
+ checkpoint = tf.train.get_checkpoint_state(ckpt_dir) 
+ input_checkpoint = checkpoint.model_checkpoint_path
+ print ("Using checkpoint=", input_checkpoint)
+ with tf.Session() as sess:
+ saver = tf.train.import_meta_graph(input_checkpoint + '.meta',
+ clear_devices=True)
+ saver.restore(sess, input_checkpoint)
+ pi = PpoBaselinesPolicy(sess)
+
+
+ for i in range(num_trials):
+
+ pi.reset()
+ ob = env.reset()
+ if len(log_header) == 0:
+ make_header(len(ob))
+
+ log_file = os.path.join(ckpt_eval_dir, "trial-{}.csv".format(i))
+
+ sim_time = 0
+ actual = np.zeros(3)
+
+ logs = []
+ while True:
+ ac = pi.action(ob, env.sim_time, env.angular_rate_sp,
+ env.imu_angular_velocity_rpy)
+ ob, reward, done, _ = env.step(ac)
+
+ log = ([env.sim_time] +
+ ob.tolist() + # The observations are the NN input
+ ac.tolist() + # The actions are the NN output
+ env.y.tolist() + # Y is the output sent to the ESC
+
+ env.imu_angular_velocity_rpy.tolist() + # Angular velocites
+ env.angular_rate_sp.tolist() + #
+ env.esc_motor_angular_velocity.tolist() +
+ [reward])# The reward that would have been given for the action, can be helpful for debugging
+
+ logs.append(log)
+
+ if done:
+ break
+ np.savetxt(log_file, logs, delimiter=",", header=log_header)
+
+ monitor = CheckpointMonitor(args.ckpt_dir, callback)
+ monitor.start()

examples/ppo_example.py → examples/ppo_baselines_train.py

@@ -27,7 +27,7 @@ def get_commit_hash():
 
 def get_training_name():
  now = datetime.datetime.now()
- timestamp = now.strftime('%y%m%d-%H%M%S')
+ timestamp = now.strftime('%Y%m%d-%H%M%S')
  return "baselines_" + get_commit_hash() + "_" + timestamp
 
 
@@ -80,19 +80,20 @@ def policy_fn(name, ob_space, ac_space):
 
 if __name__ == '__main__':
  parser = argparse.ArgumentParser("Synthesize a neuro-flight controller.")
- parser.add_argument('--checkpoint_dir', default="../../checkpoints")
+ parser.add_argument('--model_dir', default="../../models", 
+ help="Directory where models are saved to.")
  parser.add_argument('--twin', default="./gymfc_nf/twins/nf1/model.sdf",
  help="File path of the aircraft digitial twin/model SDF.")
  parser.add_argument('--gym-id', default="gymfc_nf-step-v1")
  parser.add_argument('--timesteps', type=int, default=10e6)
  parser.add_argument('--ckpt-freq', type=int, default=100e3)
  args = parser.parse_args()
 
- training_dir = os.path.join(args.checkpoint_dir, get_training_name())
- print ("Storing results to ", training_dir)
+ training_dir = os.path.join(args.model_dir, get_training_name())
 
  seed = np.random.randint(0, 1e6) 
- ckpt_dir = os.path.join(training_dir, "checkpoints")
+ ckpt_dir = os.path.abspath(os.path.join(training_dir, "checkpoints"))
+ print ("Saving checkpoints to {}.".format(ckpt_dir))
  render = False
  # How many timesteps until a checkpoint is saved
  ckpt_freq = args.ckpt_freq