DDPG with pendulum.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import gym\n",
    "import tensorflow as tf\n",
    "#tf.enable_eager_execution()\n",
    "from tensorflow.keras.layers import Dense, Input\n",
    "from tensorflow.keras import Model\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow import keras\n",
    "import tensorflow\n",
    "#tf.enable_eager_execution()\n",
    "\n",
    "from tensorflow.keras.layers import Conv1D, Flatten\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "problem='Pendulum-v0'\n",
    "env=gym.make(problem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_states=env.observation_space.shape[0]\n",
    "num_actions=env.action_space.shape[0]\n",
    "upper_bound=env.action_space.high\n",
    "lower_bound=env.action_space.low"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.4693294], dtype=float32)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.action_space.sample()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([-0.83834156, -0.54514533,  0.06397042]),\n",
       " -6.6513032903312705,\n",
       " False,\n",
       " {})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.reset()\n",
    "env.render()\n",
    "env.step(env.action_space.sample())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3, 1, array([2.], dtype=float32), array([-2.], dtype=float32))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_states,num_actions,upper_bound,lower_bound"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OUActionNoise():\n",
    "    def __init__(self,mean,std_deviation,theta=0.15,dt=1e-2,x_initial=None):\n",
    "        self.theta=theta\n",
    "        self.std_dev=std_deviation\n",
    "        self.mean=mean\n",
    "        self.dt=dt\n",
    "        self.x_initial=x_initial\n",
    "        self.reset()\n",
    "        \n",
    "    def __call__(self):\n",
    "        x=(self.x_prev+\n",
    "           self.theta*(self.mean-self.x_prev)*self.dt+\n",
    "          self.std_dev*np.sqrt(self.dt)*np.random.normal(size=self.mean.shape))\n",
    "        self.x_prev=x\n",
    "        return x\n",
    "    \n",
    "    def reset(self):\n",
    "        if self.x_initial is not None:\n",
    "            self.x_prev=self.x_initial\n",
    "            \n",
    "        else:\n",
    "            self.x_prev=np.zeros_like(self.mean)\n",
    "            \n",
    "    \n",
    "class Buffer:\n",
    "    def __init__(self,buffer_capacity=100000,batch_size=64):\n",
    "        self.buffer_capacity=buffer_capacity\n",
    "        self.batch_size=batch_size\n",
    "        self.buffer_counter=0\n",
    "        self.state_buffer=np.zeros((self.buffer_capacity,num_states))\n",
    "        self.action_buffer=np.zeros((self.buffer_capacity,num_actions))\n",
    "        self.reward_buffer=np.zeros((self.buffer_capacity,1))\n",
    "        self.next_state_buffer=np.zeros((self.buffer_capacity,num_states))\n",
    "        \n",
    "    def record(self,obs_tuple):\n",
    "        index=self.buffer_counter%self.buffer_capacity\n",
    "        self.state_buffer[index]=obs_tuple[0]\n",
    "        self.action_buffer[index]=obs_tuple[1]\n",
    "        self.reward_buffer[index]=obs_tuple[2]\n",
    "        self.next_state_buffer[index]=obs_tuple[3]\n",
    "        self.buffer_counter+=1\n",
    "        \n",
    "    @tf.function\n",
    "    def update(self,state_batch,action_batch,reward_batch,next_state_batch):\n",
    "        with tf.GradientTape() as tape:\n",
    "            target_actions=target_actor(next_state_batch,training=True)\n",
    "            y=reward_batch+gamma*target_critic([next_state_batch,target_actions],training=True)\n",
    "            critic_value=critic_model([state_batch,action_batch],training=True)\n",
    "            critic_loss=tf.math.reduce_mean(tf.math.square(y-critic_value))\n",
    "            critic_grad=tape.gradient(critic_loss,critic_model.trainable_variables)\n",
    "            critic_optimizer.apply_gradients(zip(critic_grad,critic_model.trainable_variables))\n",
    "            \n",
    "            \n",
    "    def learn(self):\n",
    "        record_range=min(self.buffer_counter,self.buffer_capacity)\n",
    "        batch_indices=np.random.choice(record_range,self.batch_size)\n",
    "        state_batch=tf.convert_to_tensor(self.state_buffer[batch_indices])\n",
    "        action_batch=tf.convert_to_tensor(self.action_buffer[batch_indices])\n",
    "        reward_batch=tf.convert_to_tensor(self.reward_buffer[batch_indices])\n",
    "        reward_batch=tf.cast(reward_batch,dtype=tf.float32)\n",
    "        next_state_batch=tf.convert_to_tensor(self.next_state_buffer[batch_indices])\n",
    "        self.update(state_batch,action_batch,reward_batch,next_state_batch)\n",
    "        \n",
    "@tf.function\n",
    "def update_target(target_weights,weights,tau):\n",
    "    for (a,b) in zip(target_weights,weights):\n",
    "        a.assign(b*tau+a*(1-tau))\n",
    "        \n",
    "def get_actor():\n",
    "    last_init=tf.random_uniform_initializer(minval=-0.003,maxval=0.003)\n",
    "    inputs=layers.Input(shape=(num_states,))\n",
    "    out=layers.Dense(256,activation='relu')(inputs)\n",
    "    out=layers.Dense(256,activation='relu')(out)\n",
    "    outputs=layers.Dense(1,activation='tanh',kernel_initializer=last_init)(out)\n",
    "    outputs= outputs * upper_bound\n",
    "    model=tf.keras.Model([inputs],outputs)\n",
    "    return model\n",
    "\n",
    "def get_critic():\n",
    "    state_input=layers.Input(shape=(num_states))\n",
    "    state_out=layers.Dense(16,activation='relu')(state_input)\n",
    "    state_out=layers.Dense(32,activation='relu')(state_out)\n",
    "    action_input=layers.Input(shape=(num_actions))\n",
    "    action_out=layers.Dense(32,activation='relu')(action_input)\n",
    "    concat=layers.concatenate([state_out,action_out])\n",
    "    out=layers.Dense(256,activation='relu')(concat)\n",
    "    out=layers.Dense(256,activation='relu')(out)\n",
    "    outputs=layers.Dense(1)(out)\n",
    "    \n",
    "    model=tf.keras.Model([state_input,action_input],outputs)\n",
    "    return model\n",
    "\n",
    "def policy(state,noise_object):\n",
    "    sampled_actions=tf.squeeze(actor_model(state))\n",
    "    noise=noise_object()\n",
    "    sampled_actions=sampled_actions.numpy()+noise\n",
    "    legal_action=np.clip(sampled_actions,lower_bound,upper_bound)\n",
    "    return [np.squeeze(legal_action)]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "std_dev=0.2\n",
    "ou_noise=OUActionNoise(mean=np.zeros(1),std_deviation=float(std_dev)*np.ones(1))\n",
    "actor_model=get_actor()\n",
    "critic_model=get_critic()\n",
    "\n",
    "target_actor=get_actor()\n",
    "target_critic=get_critic()\n",
    "\n",
    "target_actor.set_weights(actor_model.get_weights())\n",
    "target_critic.set_weights(critic_model.get_weights())\n",
    "\n",
    "critic_lr=0.002\n",
    "actor_lr=0.001\n",
    "\n",
    "critic_optimizer=tf.keras.optimizers.Adam(critic_lr)\n",
    "actor_optimizer=tf.keras.optimizers.Adam(actor_lr)\n",
    "\n",
    "total_episodes=10\n",
    "gamma=0.99\n",
    "tau=0.005\n",
    "buffer=Buffer(50000,64)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode * 0 * Avg Reward is ==> -1910.969190308048\n",
      "Episode * 1 * Avg Reward is ==> -1607.6062358728398\n",
      "Episode * 2 * Avg Reward is ==> -1418.160833310759\n",
      "Episode * 3 * Avg Reward is ==> -1420.6982729489334\n",
      "Episode * 4 * Avg Reward is ==> -1406.3952955265654\n",
      "Episode * 5 * Avg Reward is ==> -1349.6560991927672\n",
      "Episode * 6 * Avg Reward is ==> -1311.3878893827928\n",
      "Episode * 7 * Avg Reward is ==> -1304.7940261796787\n",
      "Episode * 8 * Avg Reward is ==> -1274.9243257165804\n",
      "Episode * 9 * Avg Reward is ==> -1294.3294316099643\n"
     ]
    }
   ],
   "source": [
    "ep_reward_list=[]\n",
    "avg_reward_list=[]\n",
    "\n",
    "for ep in range(total_episodes):\n",
    "    prev_state=env.reset()\n",
    "    episodic_reward=0\n",
    "    \n",
    "    while True:\n",
    "        tf_prev_state=tf.expand_dims(tf.convert_to_tensor(prev_state),0)\n",
    "        action=policy(tf_prev_state,ou_noise)\n",
    "        env.render()\n",
    "        state, reward, done, info = env.step(action)\n",
    "        buffer.record((prev_state,action,reward,state))\n",
    "        episodic_reward+=reward\n",
    "        buffer.learn()\n",
    "        update_target(target_actor.variables,actor_model.variables,tau)\n",
    "        update_target(target_critic.variables,critic_model.variables,tau)\n",
    "        \n",
    "        if done:\n",
    "            break\n",
    "            \n",
    "        prev_state=state\n",
    "        \n",
    "    ep_reward_list.append(episodic_reward)\n",
    "    avg_reward=np.mean(ep_reward_list[-40:])\n",
    "    print(\"Episode * {} * Avg Reward is ==> {}\".format(ep, avg_reward))\n",
    "    avg_reward_list.append(avg_reward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEGCAYAAACgt3iRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAArTklEQVR4nO3deXxc9Xnv8c9jLZZkS7Zly5Ysr3g3NrZBZQlL2JewGAiEQBpomkJSQpI2TQMJ997Sm942pdkISUncBEKaNiSlsc1uwAFMw2qQjBd5w3jRWLLlVfIia5nn/jFHZmwkeezR6Iw03/frNa8553fOmfNoXrYe/Zbz+5m7IyIikox+YQcgIiK9n5KJiIgkTclERESSpmQiIiJJUzIREZGkZYcdQFiGDRvm48aNCzsMEZFe5Z133tnh7iVHl2dsMhk3bhxLly4NOwwRkV7FzDZ1VK5mLhERSZqSiYiIJE3JREREkqZkIiIiSVMyERGRpCmZiIhI0pRMREQkaRn7nImISE9oaYtSuXkP727ezYii/kwtLWJCyUBys/vW3/JKJiIi3cjd2bTzAK+uq+eVtTt4Y8NO9h1qPeKc7H7GhJKBTC0rZEppIdNKi5haVkhpUR5mFlLkyVEyERFJUkNTC6+t38mSdfW8uq6eLbsOAjBqSD7XzB7JeZOGcfr4oezYd4jVdY2srm1gdV0jSzfuZmHV1sOfU5SXzdSyIqaVFjIlSDBTRhQyoH/6/6pO/whFRNJMa1uU9yJ7WbK2nlfX7aBqyx7aos6A3CzOmjCM2889ifMmlTB2aMERNY3iAblMHlHINbNGHi7be7CFNXWNrKlroDpINI+/U8P+5rbD54wpLmBqaSFTy4pi76WFjB06gKx+6VOLUTIREUlAze4DvLpuB0vW1vPH9TtoaGrFDE4pH8RffnwC500uYc6YweRkHV9fyKD8HE4fX8zp44sPl0WjTmTPQaprG1hT18jqukaq6xp4sXob0WCl9bycfkweURgkl6LDyaZ4QG53/tgJs0xdA76iosI10aOIdGb/oVbe2LDzcO1jw479AJQNyuPcScM4d1IJ50wcxpAe/OXd1NLGum37qK5rTzINrK5tZOf+5sPnDC/sH+uHCWoxU0oLmTh8IP2zs7olBjN7x90rji4PpWZiZjcC9wHTgNPdfWlQfjowr/004D53nx8cuxx4AMgCfu7u3wnKxwOPAUOBd4DPuvuH36yISAKiUWfl1gaWrKtnydp63t28m5Y2Jy+nH2eeNJTPnDmWj08exoSSgaF1kuflZDFz1CBmjhp0RHl94yFWBwmmujaWZH752kaaW6MAZPUzJpQMiPXDlBbyqYrRlBT279bYQqmZmNk0IAr8DPh6XDIpAJrdvdXMyoBlwEjAgbXAJUAN8DZws7uvMrPfAb9398fM7KfAMnd/6FgxqGYiInV7m4JO8x38z7p6dh9oAWB6WRHnTh7GxyeVcNq4Id32V31Pam2LsnHnfqprGw/XYqprG4nsOcir37iA0cUFJ/S5aVUzcfdq4CPZ3d0PxO3mEUsiAKcD6919Q3DdY8BcM6sGLgRuCc57lFiN55jJREQyz8HmNt7auCtouqpn7bZ9AAwb2J8Lpgzn3MnDOGdiSbf/1R6G7Kx+TBxeyMThhVw968PyhqYWClMwOiztOuDN7AzgYWAssSarVjMrB7bEnVYDnEGsaWuPu7fGlZd38dl3AHcAjBkzJgXRi0g6aGppo7GplYamFvYcaGbpxt28um4Hb23cRXNrlNzsfpw+rphPnjqK8yaXMLW0sNc+33G8ivJyUvK5KUsmZvYiUNrBoXvdfWFn17n7m8DJQVPYo2b2bHfF5O7zCPpkKioqMnPkgUiai0adfc2tNBxsiSWEgy00NLXS2NTyYVnTUe9x5Q1NrYf7CuJNHjGQz545lvMml3D6uGLyc3tf01U6S1kycfeLk7y+2sz2ATOACDA67vCooGwnMNjMsoPaSXu5iIRs9/5mNu06ECSB9l/8se3GpvgEcWRy2HeolWN15ebnZFGYl01Rfg6FedkMLshldHHB4f2ivByK4o5PLxtE6aC8nvnBM1RaNXMFI7O2BE1bY4GpwEZgDzApOB4BPg3c4u5uZi8BNxAb0XUb0GmtR0RSa9+hVl5YVcfCqq28um4HbdGPZoV+xhG/9AvzshlTXEBhXg5F+dmx9/aEcHg/54jkcbzPckjqhTU0+DrgQaAEeNrMqtz9MuAc4B4zayE22utOd98RXHMXsIjY0OCH3X1l8HF3A4+Z2T8AlcAvevanEclsza1RXllbz8KqCC9Wb6OpJUr54HzuOO8kKsYOYVB+zuFEUZSXQ0FuVsb0T2QSPbQoIsctGnXe/GAXTyyL8MzyOvYebKF4QC5Xzixj7uyRnDpmCP3SaKoP6T5pNTRYRHof99hDfQurIjy5rJa6hiYKcrO47ORSrpk9knMmDlPzUwZTMhGRLn2wYz9PVG1l4bIIG+r3k5NlfHzycO69choXTxuhUVECKJmISAe2NzTx5Hu1PFEVYVnNXszgjPHF3H7uSVwxo5TBBeFMJijpS8lERIDYVOiLVtSxcFmE19/fSdRhRnkR935iGlfNKqNsUH7YIUoaUzIRyWBNLW28tHo7C6oivLS6nua2KGOHFnDXBRO5ZnY5E4cPDDtE6SWUTEQyTGtblNc37GRh1VYWraij8VArwwb25zNnjmHu7HJmjRqkobty3JRMRDKAu1O1ZQ8Lq7by1Hu17Nh3iML+2Vw+o5S5s8s5a8LQtFq1T3ofJRORPmz99kYWVm1lYdVWNu86QG52Py6aOpy5s0dy/pTh5OVoJJZ0DyUTkT6mvvEQv3+3hoVVW1lV20A/g7MnDuPLF07kshmlKZs1VjKbkolIH7HnQDM/W7KBX/5xIwdb2pg9ejB/d/V0rjyljOGFmuRQUkvJRKSX23eolUf+5wPmvbqBfYdauWbWSL584SSNxJIepWQi0ks1tbTx6zc28a8vv8+u/c1cMn0Ef3PpZKaWFoUdmmQgJRORXqalLcp/La3hR4vXUdfQxDkTh/E3l05mzpghYYcmGUzJRKSXaIs6Ty7byg9eXMumnQc4dcxgvn/TLD42YVjYoYkomYikO3dn0cptfP+FNazdto9pZUU8/GcVXDBluB4ulLShZCKSptydV9ft4HvPr2FZzV5OGjaAH98yh0/MKNNaIZJ2lExE0tDSjbv4l0VrePODXZQPzuf+T57C9aeWk631QiRNKZmIpJEVkb187/k1vLSmnmED+3Pf1dO5+Ywx9M/Wk+qS3pRMRNLA+u37+MELa3l6eS2D8nP4xuVT+LOPjaMgV/9FpXfQv1SREG3ZdYAHFq/j9+/WkJeTxZcvnMhfnHsSg/I15Yn0LkomIiHY3tDEj19az2/e2oyZ8bmzx/OX509g2MD+YYcmckKUTER60O79zfx0yfs8+tpGWtucGytG85WLJmoVQ+n1QhkaYmY3mtlKM4uaWUUHx8eY2T4z+3pc2eVmtsbM1pvZPXHl483szaD8t2amxakl7TQ2tfDAi+s47/6XmLdkA1fMKGPx33ycf7p+phKJ9Alh1UxWANcDP+vk+PeBZ9t3zCwL+AlwCVADvG1mT7j7KuCfgR+4+2Nm9lPg88BDqQxeJFFNLW38++ub+NeX17P7QAuXnTyCr10yhSmlhWGHJtKtQkkm7l4NdPj0rpldC3wA7I8rPh1Y7+4bgnMeA+aaWTVwIXBLcN6jwH0omUjImluj/HbpFn78h3VsazjEuZOG8fVLpzBr9OCwQxNJibTqMzGzgcDdxGogX487VA5siduvAc4AhgJ73L01rry8i8+/A7gDYMyYMd0XuEigLeosqIzww8Vr2bLrIBVjh/DAp+dw5klDww5NJKVSlkzM7EWgtIND97r7wk4uu49Yk9W+VMw55O7zgHkAFRUV3u03kIzS1NJGze6DbNl1gC27D7B55wFeXlvP+u37OHlkEY/82QzOn1Ki+bMkI6Qsmbj7xSdw2RnADWZ2PzAYiJpZE/AOMDruvFFABNgJDDaz7KB20l4ukrRo1NneeOhwotgcJI0tu2Lb2xoOHXF+/+x+TCkt5Ce3nMoVM0o1f5ZklLRq5nL3c9u3zew+YJ+7/9jMsoFJZjaeWLL4NHCLu7uZvQTcADwG3AZ0VusR+YjGpha27DrI5l0HqNkdSxKbdx0IahsHaW6NHj7XDEqL8hhdXMA5E0sYU1zAmKH5jB5SwJjiAoYN7K8EIhkrlGRiZtcBDwIlwNNmVuXul3V2vru3mtldwCIgC3jY3VcGh+8GHjOzfwAqgV+kNnrpTVraotTuaYrVLuITRbC9+0DLEecX9s9mzNACJg0v5KJpIxhdXMDoIfmMKS6gfEi+5sgS6YS5Z2bXQUVFhS9dujTsMHq17Y1NfOqnrxPZc5CcrH6HX7lZRk72Ufvtx7OP2s/qR262ffT64NxErweI7Gk6IlFs2X2ArXuaaIt++G88u59RHiSHWKKI1Spi+/kMys9RH4dIF8zsHXf/yPOBadXMJb3Lvzy3hsieg/z52eNpizotbVGa22Lv7a/m1g/3W9ucgwdb4o47za3RI/fbokc0LZ2IYQNzGV1cwJzRQ5g7K5YoRhXHEkhpUZ6mcRdJASUTOSHLa/by+Ls1/MU54/nmJ6Z162e7e5CcYsnlcLJpPWo/LllF3SkblM+oIfkM6K9/1iI9Tf/r5Li5O3//5EqKC3L58kWTuv3zzYzsLCM7C/JRH4VIb9BpMjGzB4FOO1Tc/SspiUjS3lPv1bJ0027+8bqZFOVpqnQR6Xqix6XEnu/IA04F1gWv2YAmU8xQTS1tfOfZ1UwrK+KmPxl97AtEJCN0WjNx90cBzOwvgXPapywJJlN8tWfCk3Qzb8kGInsO8t0bZ5GlZypEJJDIsJYhQFHc/sCgTDJM3d4mHnr5fS4/uZSzJmiuKRH5UCId8N8BKoMnzQ04j9gcWpJh/vm51bS5861uHr0lIr1fl8nEzPoBa4jNmXVGUHy3u9elOjBJL+9u3s38ygh3nj+BMUMLwg5HRNJMl8nE3aNm9hN3n4PmvMpY0ajzf59cRUlhf+68YGLY4YhIGkqkz2SxmX3SNMdExlq4LELVlj1847IpDNQDgSLSgUSSyReA/wIOmVmDmTWaWUOK45I0caC5lX9+dg2njBrEJ08dFXY4IpKmjvlnprtrseoM9tOX36euoYkf3zJH06uLSKcSarMwsyHAJGIPMALg7ktSFZSkh5rdB/jZkg1cPWskFeOKww5HRNLYMZOJmf0F8FViqxhWAWcCrwMXpjQyCd13nl2NGdxzxdSwQxGRNJdIn8lXgT8BNrn7BcAcYE8qg5Lwvb1xF0+9V8sd502gfHB+2OGISJpLJJk0uXsTgJn1d/fVwJTUhiVhah8KXFqUxxc/flLY4YhIL5BIn0mNmQ0GFgAvmNluYFMqg5JwPf5uDcsje/nhTbMpyNVQYBE5tkRGc10XbN4XTKkyCHgupVFJaPYdauVfFq1hzpjBzJ09MuxwRKSXSKQD/tvAEuA1d38l9SFJmH7y0nrqGw/xb7dWaC10EUlYIn0mG4CbgaVm9paZfc/M5qY4LgnB5p0H+MWrH3D9nHJmjx4cdjgi0oscM5m4+yPu/ufABcCvgRuDd+lj/t8zq8jqZ3zjcg0FFpHjc8xkYmY/N7PXgIeINYvdQJLrmZjZjWa20syiZlYRVz7OzA6aWVXw+mncsdPMbLmZrTezH7XPFWZmxWb2gpmtC9611soJeO39HSxauY07z59A6aC8Y18gIhInkWauoUAWsWdLdgE72lddTMIK4HpifTFHe9/dZwevL8aVPwTcTuxJ/EnA5UH5PcBid58ELA725Ti0BUOBywfnc/t5GgosIscvkWau69z9DOB+YDDwkpnVJHNTd6929zWJnm9mZUCRu7/h7g78Crg2ODwXeDTYfjSuXBL02NubWV3XyDc/MZW8nKywwxGRXiiR0VxXAecSW2FxMPAHUrsG/HgzqwQagP/l7q8C5UB8AqsJygBGuHttsF0HjOjsg83sDuAOgDFjxnR33L3S3oMtfO/5tZw+rpgrZ5aFHY6I9FKJPJF2ObHk8YC7b030g83sRaC0g0P3untnC23VAmPcfaeZnQYsMLOTE72nu7uZeRfH5wHzACoqKjo9L5M8uHgduw8083+unq6hwCJywhJ5aPEuMxsLTAe2mlk+kO3ujce47uLjDcbdDwGHgu13zOx9YDIQITbRZLtRQRnANjMrc/faoDls+/HeN1NtqN/HL1/byKdOG82M8kFhhyMivVgio7luBx4HfhYUjSI2tUq3M7MSM8sKtk8i1tG+IWjGajCzM4NRXLfy4TLCTwC3Bdu3oeWFE/b/nq4mLyeLr1+mqdZEJDmJjOb6EnA2sT4M3H0dMDyZm5rZdUEn/lnA02a2KDh0HvCemVURS2BfdPddwbE7gZ8D64H3gWeD8u8Al5jZOuDiYF+OYcnaehav3s5dF06kpLB/2OGISC+XSJ/JIXdvbm9PN7NsIKn+BnefD8zvoPy/gf/u5JqlwIwOyncCFyUTT6ZpbYvy7adWMXZoAZ87e1zY4YhIH5BIzeQVM/sWkG9mlxBbD/7J1IYlqfQfb25m3fZ9fOsT0+ifraHAIpK8RJLJPUA9sBz4AvCMu9+b0qgkZfYcaOYHL67lYxOGcun0TkdRi4gcl0QeWoy6+7+5+43ufgOwycxe6IHYJAV++OI6Gg62aCiwiHSrTpOJmV1oZmvNbJ+Z/drMZprZUuCfiE1tIr3Mum2N/Psbm7j59DFMLS0KOxwR6UO6qpl8j9jT4kOJjax6Hfilu5/m7r/vieCk+7g73366moLcLL52yeSwwxGRPqarZOLu/rK7H3L3BUDE3X/cQ3FJN3tpzXaWrK3nqxdNYuhADQUWke7V1dDgwWZ2ffy58fuqnfQeza1R/uGpak4aNoBbzxoXdjgi0gd1lUxeAa6O218St++Akkkv8avXN7Jhx34e/rMKcrMTGcAnInJ8Ok0m7v65ngxEUmPnvkM8sHgd500u4YIpSU1cICLSKf2Z2sd9/4W1HGhu439fOU1DgUUkZZRM+rDq2gZ+89ZmPnvmWCaNKAw7HBHpw5RM+ih359tPraIoP4e/unhS2OGISB+XyBT0XzKzwXH7Q8zszpRGJUl7ftU2Xnt/J1+7ZDKDC3LDDkdE+rhEaia3u/ue9h133w3cnrKIJGmHWtv4x2eqmTxiILecruWJRST1EkkmWRbXcxssXqU/ddPYI3/cyKadB/jfV00nO0stmSKSeomsZ/Ic8Fsza19p8QtBmaSh7Y1NPLh4HRdPG865k0rCDkdEMkQiyeRuYgnkL4P9F4iteChp6LuL1tDcFuXeK6eHHYqIZJBjJhN3jxKbJVgzBae5FZG9/Nc7NfzFOeMZP2xA2OGISAbpNJmY2e/c/VNmtpwOlul191NSGpkcF3fn759cSXFBLl++SEOBRaRndVUz+WrwflVPBCLJeXp5LW9v3M0/XjeTorycsMMRkQzT1dxctcH7pp4LR05EU0sb//TMaqaVFXHTn4wOOxwRyUBdNXM10kHzVjt311J9aeLflmwgsucg371xFln9NP+WiPS8Th9CcPfCIGE8ANwDlAOjiI3u+mEyNzWzG81spZlFzaziqGOnmNnrwfHlZpYXlJ8W7K83sx+1P/tiZsVm9oKZrQvehyQTW29Tt7eJf335fS4/uZSzJgwNOxwRyVCJPNF2jbv/q7s3unuDuz8EzE3yviuA64mtkXKYmWUDvwa+6O4nA+cDLcHhh4g9eT8peF0elN8DLHb3ScDiYD9j3P/catqizrc+MS3sUEQkgyWSTPab2WfMLMvM+pnZZ4D9ydzU3avdfU0Hhy4F3nP3ZcF5O929zczKgCJ3f8PdHfgVcG1wzVzg0WD70bjyPq9y825+Xxnh8+eOZ8zQgrDDEZEMlkgyuQX4FLAN2A7cGJSlwmTAzWyRmb1rZt8IysuBmrjzaoIygBHtgwWAOmBEZx9uZneY2VIzW1pfX9/dsfcod+f/PrWKksL+fOmCiWGHIyIZLpGHFjdyAs1aZvYiUNrBoXvdfWEX8ZwD/AlwAFhsZu8AexO5p7u7mXU1aGAeMA+goqKi0/N6g4VVW6ncvIf7bziFgf0TmchARCR1jvlbyMxGAQ8CZwdFrwJfdfeazq8Cd7/4BOKpAZa4+47g3s8ApxLrRxkVd94oIBJsbzOzMnevDZrDtp/AfXuVaNT57vNrmFk+iBtOHXXsC0REUiyRZq5HgCeAkcHryaAsFRYBM82sIOiM/ziwKmjGajCzM4NRXLcC7bWbJ4Dbgu3b4sr7rLc27qJm90FuP+8k+mkosIikgUSSSYm7P+LurcHrl0BS09Ga2XVmVgOcBTxtZovg8Fop3wfeBqqAd9396eCyO4lNMLkeeB94Nij/DnCJma0DLg72+7QFlREG5GZxybROu4dERHpUIo3tO83sT4HfBPs3AzuTuam7zwfmd3Ls18SatY4uXwrM6KB8J3BRMvH0Jk0tbTy9vJbLZpSSn5sVdjgiIkBiNZM/Jzaaqw6oBW4APpfKoKRzL63eTmNTK9fNKT/2ySIiPSSR0VybgGt6IBZJwIKqCCWF/fnYhGFhhyIiclhXc3N9w93vN7MH6XgK+q+kNDL5iD0HmnlpdT23njVWc3CJSFrpqmZSHbwv7YlA5NieWV5Hc1uUa9XEJSJppqsp6J8M3tunKsHM+gED3b2hB2KToyyojDBx+EBOHqkJm0UkvRyzA97M/tPMisxsALEJGleZ2d+mPjSJt2XXAd7auIvr5pQTTJgsIpI2EhnNNT2oiVxL7NmO8cBnUxmUfNQTy7YCcM2skSFHIiLyUYkkkxwzyyGWTJ5w9xa6WDRLup+7M78ywunjihldrNmBRST9JJJMfgZsBAYAS8xsLKA+kx60cmsD67fvU8e7iKStRJ4z+RHwo7iiTWZ2QepCkqMtqIyQm9WPK2eWhR2KiEiHEumAHxosk/uumb1jZg8Ag3ogNgHaos7CZVs5f0oJgwpywg5HRKRDiTRzPQbUA58kNpVKPfDbVAYlH3rt/R3UNx7S9CkiktYSmeixzN2/Hbf/D2Z2U6oCkiPNr4xQmJfNBVOHhx2KiEinEqmZPG9mnw7Wf+9nZp8itu6IpNjB5jYWrajjypll5OVohmARSV+JJJPbgf8EDgWvx4AvmFmjmWlUVwq9UL2N/c1tGsUlImkvkdFchT0RiHzUgsoIIwflcfq44rBDERHpUqc1k2BBrPbts486dlcqgxLYue8Qr6yt55rZ5VqaV0TSXlfNXF+L237wqGN/noJYJM5T79XSFnWN4hKRXqGrZGKdbHe0L91sfmWEaWVFTClVK6OIpL+ukol3st3RvnSjjTv2U7VlD9fN0aSOItI7dNUBP9XM3iNWC5kQbBPsn5TyyDLYgqoIZnDNLDVxiUjv0FUymdZjUchh7s6CyghnnTSU0kF5YYcjIpKQTpu53H1TV69kbmpmN5rZSjOLmllFXPlnzKwq7hU1s9nBsdPMbLmZrQ/mCrOgvNjMXjCzdcH7kGRiC1vVlj1s3HlAz5aISK+SyEOLqbACuB5YEl/o7v/h7rPdfTaxBbg+cPeq4PBDxB6gnBS8Lg/K7wEWu/skYHGw32stqIzQP7sfl88oDTsUEZGEhZJM3L3a3dcc47SbiT1tj5mVAUXu/oa7O/ArYot1AcwF2tepfzSuvNdpaYvy1Hu1XDx9BEV5miFYRHqPsGomibgJ+E2wXQ7UxB2rCcoARrh7bbBdB4zo7APN7A4zW2pmS+vr67s73qT9z7od7NzfzHWz1cQlIr3LCSUTM7svgXNeNLMVHbzmJnDtGcABd19xPHEFtZZOhy27+zx3r3D3ipKSkuP56B4xvzLC4IIczpucfrGJiHQlkSnoO/LOsU5w94tP8LMBPs2HtRKACDAqbn9UUAawzczK3L02aA7bnsR9Q7PvUCvPr6rjhtNGkZudzhVGEZGPOqHfWu7+ZHcH0s7M+gGfIugvCe5XCzSY2ZnBKK5bgYXB4SeA24Lt2+LKe5VFK+poaolq+hQR6ZWOWTMxsx91ULwXWOruJ/SL28yuIzbfVwnwtJlVuftlweHzgC3uvuGoy+4EfgnkA88GL4DvAL8zs88Dm4glol5nQVWE0cX5nDqmV49sFpEMlUgzVx4wFfivYP+TwAfALDO7wN3/6nhv6u7zgfmdHHsZOLOD8qXAjA7KdwIXHW8M6WR7QxN/XL+Duy6YSPD4jIhIr5JIMjkFONvd2wDM7CHgVeAcYHkKY8sYTyzbStRhrpq4RKSXSqTPZAgwMG5/AFAcJJdDKYkqwyyoinDKqEFMKBl47JNFRNJQIjWT+4EqM3uZ2CSP5wH/aGYDgBdTGFtGWLetkRWRBv7PVdPDDkVE5IQlsmzvL8zsGeD0oOhb7r412P7blEWWIRZURcjqZ1w9S9PNi0jvlchorieB/wSecPf9qQ8pc0SjzoLKrZwzcRglhf3DDkdE5IQl0mfyXeBcYJWZPW5mN5iZ5kbvBu9s3k1kz0E9WyIivV4izVyvAK+YWRZwIbGZex8GilIcW583vzJCQW4Wl57c6XRiIiK9QkLTqZhZPnA1sckXT+XDWXrlBB1qbePp92q5dPoICnJPdFYbEZH0kEifye+Idb4/B/wYeMXdo6kOrK97eU09ew+2aBEsEekTEvmT+BfAzXEPLZ5jZje7+5dSG1rftqAywrCBuZwzcVjYoYiIJO2YHfDuvgg4xczuN7ONwLeB1akOrC/be7CFxdXbuXrWSLKzNEOwiPR+ndZMzGwysdUObwZ2AL8FzN0v6KHY+qznVtTS3KYZgkWk7+iqmWs1sTm4rnL39QBm9tc9ElUfN78ywknDBjCzfFDYoYiIdIuu2liuB2qBl8zs38zsImLTqUgSInsO8saGXVw7p1wzBItIn9FpMnH3Be7+aWLTz78E/BUw3MweMrNLeyi+PueJqthMNNdqnXcR6UMS6YDf7+7/6e5XE1sutxK4O+WR9VELKiOcNnYIY4YWhB2KiEi3Oa6hRO6+293nuXuvXowqLNW1DazZ1qhnS0Skz9G41B60oDJCdj/jqpllYYciItKtlEx6SFvUWVi1lfOnlDBkQG7Y4YiIdCslkx7y5oad1DU0qYlLRPokJZMeMr8ywsD+2Vw8TTMEi0jfo2TSA5pa2nh2RR1XzCglLycr7HBERLpdKMnEzG40s5VmFjWzirjyHDN71MyWm1m1mX0z7tjlZrbGzNab2T1x5ePN7M2g/LdmlnYdEourt7PvUKumTxGRPiusmskKYk/YLzmq/Eagv7vPBE4DvmBm44KFuX4CXAFMB242s+nBNf8M/MDdJwK7gc/3xA9wPOZXRigtyuOMk4aGHYqISEqEkkzcvdrd13R0CBhgZtlAPtAMNBBbT2W9u29w92bgMWCuxeYjuRB4PLj+UeDaVMd/PHbtb+blNdu5ZvZIsvpp+hQR6ZvSrc/kcWA/sTnBNgPfdfddQDmwJe68mqBsKLDH3VuPKu+Qmd1hZkvNbGl9fX0q4v+Ip5fX0hp1TZ8iIn1aytaLNbMXgdIODt3r7gs7uex0oA0YCQwBXg0+p1u4+zxgHkBFRYV31+d2ZUFlhCkjCplWVtgTtxMRCUXKkom7X3wCl90CPOfuLcB2M/sjUEGsVjI67rxRQATYCQw2s+ygdtJenhY27zzAO5t2c/flUzVDsIj0aenWzLWZWB8IZjYAOJPYuipvA5OCkVu5wKeBJ9zdic1ofENw/W1AZ7WeHrewKpbX5s4eGXIkIiKpFdbQ4OvMrAY4C3jazBYFh34CDDSzlcQSyCPu/l5Q67gLWARUA79z95XBNXcDXzOz9cT6UH7Rkz9LZ9yd+VURzhhfzMjB+WGHIyKSUilr5uqKu88H5ndQvo/Y8OCOrnkGeKaD8g3E+lrSyvLIXjbU7+eOc08KOxQRkZRLt2auPmN+ZYTcrH5coRmCRSQDKJmkQGtblCeXbeWiacMZlJ8TdjgiIimnZJICf3x/Jzv2NWuGYBHJGEomKbCgMsKg/BzOn1ISdigiIj1CyaSb7T/UynMr6vjEzDL6Z2uGYBHJDEom3eyFVds42NKmGYJFJKMomXSz+ZURygfnUzF2SNihiIj0GCWTblTfeIhX19Vz7ZyR9NMMwSKSQZRMutFT720l6miGYBHJOEom3WhBZYQZ5UVMGqEZgkUksyiZdJP36/exrGavaiUikpGUTLrJwsoI/QyunqUZgkUk8yiZdIP2GYLPnjiMEUV5YYcjItLjlEy6wbubd7Nl10E1cYlIxlIy6QYLKreSl9OPy2Z0tEqxiEjfp2SSpObWKE+9t5VLppcysH8oy8OIiIROySRJS9bWs/tAC9fNUce7iGQuJZMkza+KUDwgl3MnaYZgEclcSiZJaGhq4cVV27j6lDJysvRVikjm0m/AJDy3oo5DrVEtgiUiGU/JJAkLqyKMG1rA7NGDww5FRCRUSiYnqG5vE6+9v5O5s8sx0wzBIpLZQkkmZnajma00s6iZVcSV55rZI2a23MyWmdn5ccdOC8rXm9mPLPgNbmbFZvaCma0L3ntkIZEnlkVwR01cIiKEVzNZAVwPLDmq/HYAd58JXAJ8z8zaY3woOD4peF0elN8DLHb3ScDiYD/l5lduZfbowYwfNqAnbiciktZCSSbuXu3uazo4NB34Q3DOdmAPUGFmZUCRu7/h7g78Crg2uGYu8Giw/WhcecqsqWukurZBS/OKiATSrc9kGXCNmWWb2XjgNGA0UA7UxJ1XE5QBjHD32mC7DhiR6iAXVEXI6mdcdUpZqm8lItIrpGz+DzN7Eehosqp73X1hJ5c9DEwDlgKbgNeAtkTv6e5uZt5FTHcAdwCMGTMm0Y89QjTqLKyM8PHJJQwd2P+EPkNEpK9JWTJx94tP4JpW4K/b983sNWAtsBsYFXfqKCASbG8zszJ3rw2aw7Z38fnzgHkAFRUVnSadrry1cRdb9zZx9xVTT+RyEZE+Ka2aucyswMwGBNuXAK3uvipoxmowszODUVy3Au21myeA24Lt2+LKU2JBZYQBuVlcOl0zBIuItAtlmlszuw54ECgBnjazKne/DBgOLDKzKLGax2fjLrsT+CWQDzwbvAC+A/zOzD5PrGnsU6mMfezQAdz6sXHk52al8jYiIr2KxQZHZZ6KigpfunRp2GGIiPQqZvaOu1ccXZ5WzVwiItI7KZmIiEjSlExERCRpSiYiIpI0JRMREUmakomIiCRNyURERJKmZCIiIknL2IcWzaye2BPzJ2IYsKMbw+nt9H18SN/FkfR9HKkvfB9j3b3k6MKMTSbJMLOlHT0Bmqn0fXxI38WR9H0cqS9/H2rmEhGRpCmZiIhI0pRMTsy8sANIM/o+PqTv4kj6Po7UZ78P9ZmIiEjSVDMREZGkKZmIiEjSlEyOk5ldbmZrzGy9md0TdjxhMbPRZvaSma0ys5Vm9tWwY0oHZpZlZpVm9lTYsYTNzAab2eNmttrMqs3srLBjCouZ/XXw/2SFmf3GzPLCjqm7KZkcBzPLAn4CXAFMB242s+nhRhWaVuBv3H06cCbwpQz+LuJ9FagOO4g08QDwnLtPBWaRod+LmZUDXwEq3H0GkAV8Otyoup+SyfE5HVjv7hvcvRl4DJgbckyhcPdad3832G4k9ouiPNyowmVmo4ArgZ+HHUvYzGwQcB7wCwB3b3b3PaEGFa5sIN/MsoECYGvI8XQ7JZPjUw5siduvIcN/gQKY2ThgDvBmyKGE7YfAN4BoyHGkg/FAPfBI0Oz3czMbEHZQYXD3CPBdYDNQC+x19+fDjar7KZlIUsxsIPDfwF+5e0PY8YTFzK4Ctrv7O2HHkiaygVOBh9x9DrAfyMg+RjMbQqwFYzwwEhhgZn8ablTdT8nk+ESA0XH7o4KyjGRmOcQSyX+4++/DjidkZwPXmNlGYs2fF5rZr8MNKVQ1QI27t9dWHyeWXDLRxcAH7l7v7i3A74GPhRxTt1MyOT5vA5PMbLyZ5RLrRHsi5JhCYWZGrD282t2/H3Y8YXP3b7r7KHcfR+zfxR/cvc/99Zkod68DtpjZlKDoImBViCGFaTNwppkVBP9vLqIPDkbIDjuA3sTdW83sLmARsREZD7v7ypDDCsvZwGeB5WZWFZR9y92fCS8kSTNfBv4j+MNrA/C5kOMJhbu/aWaPA+8SGwVZSR+cVkXTqYiISNLUzCUiIklTMhERkaQpmYiISNKUTEREJGlKJiIikjQlE5FuYmZtZlYV9+ryiW8z+6KZ3doN991oZsOS/RyRZGhosEg3MbN97j4whPtuJDYj7Y6evrdIO9VMRFIsqDncb2bLzewtM5sYlN9nZl8Ptr8SrA3znpk9FpQVm9mCoOwNMzslKB9qZs8H62P8HLC4e/1pcI8qM/tZsGyCSMopmYh0n/yjmrluiju2191nAj8mNrvw0e4B5rj7KcAXg7K/ByqDsm8BvwrK/w74H3c/GZgPjAEws2nATcDZ7j4baAM+050/oEhnNJ2KSPc5GPwS78hv4t5/0MHx94hNPbIAWBCUnQN8EsDd/xDUSIqIrRNyfVD+tJntDs6/CDgNeDs2BRT5wPYkfh6RhCmZiPQM72S73ZXEksTVwL1mNvME7mHAo+7+zRO4ViQpauYS6Rk3xb2/Hn/AzPoBo939JeBuYBAwEHiVoJnKzM4HdgRrxiwBbgnKrwCGBB+1GLjBzIYHx4rNbGzqfiSRD6lmItJ98uNmUIbY+uftw4OHmNl7wCHg5qOuywJ+HSx1a8CP3H2Pmd0HPBxcdwC4LTj/74HfmNlK4DViU5zj7qvM7H8BzwcJqgX4ErCpm39OkY/Q0GCRFNPQXckEauYSEZGkqWYiIiJJU81ERESSpmQiIiJJUzIREZGkKZmIiEjSlExERCRp/x/X7Ml+sCyLBwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(avg_reward_list)\n",
    "plt.xlabel(\"Episode\")\n",
    "plt.ylabel(\"Avg. Epsiodic Reward\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}