fees_optimization_using_RL.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "fees_optimization_using_RL",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyOX4dtRsRRnwoc86RJu7LF/",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/pranshurastogi29/Miner_fees_optimization_RL/blob/main/fees_optimization_using_RL.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B7flJ4PPWTPU"
      },
      "source": [
        "!pip -q install stable-baselines3[extra]"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RLLAFocVzSh8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ac67e8ee-1d78-4516-feb2-750dcc8847e4"
      },
      "source": [
        "import math\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import gym\n",
        "from gym import spaces\n",
        "from gym.utils import seeding\n",
        "\n",
        "from stable_baselines3 import PPO\n",
        "from stable_baselines3.common.vec_env import DummyVecEnv\n",
        "from stable_baselines3.common.evaluation import evaluate_policy\n",
        "from stable_baselines3.common.callbacks import EvalCallback, StopTrainingOnRewardThreshold\n",
        "from stable_baselines3 import DQN\n",
        "\n",
        "class FeesMaximizationEnv(gym.Env):\n",
        "    \"\"\"\n",
        "    Description:\n",
        "        The agent (a miner) want to select the transaction. For any given\n",
        "        state the agent may choose to accept or reject the transaction keeping \n",
        "        weight is less then limit\n",
        "    \n",
        "    Observation:\n",
        "        Type: Box(2)\n",
        "        Num    Observation               Min            Max\n",
        "        0      Fee                       0              inf\n",
        "        1      Weights                   0              4000000\n",
        "    Actions:\n",
        "        Type: Discrete(3)\n",
        "        Num    Action\n",
        "        0      Dont process the transaction\n",
        "        1      Process the transaction\n",
        "\n",
        "    Reward:\n",
        "         Reward of 1 is awarded if the running average fee increases\n",
        "         Reward of -1 is awarded if the running average fee decreases\n",
        "\n",
        "    Episode Termination:\n",
        "         Weight exceeds 4000000\n",
        "    \"\"\"\n",
        "\n",
        "    metadata = {\n",
        "        'render.modes': ['human', 'rgb_array'],\n",
        "        'video.frames_per_second': 30\n",
        "    }\n",
        "\n",
        "    def __init__(self, goal_velocity=0):\n",
        "        self.min_fee = 0\n",
        "        self.max_fee =  np.finfo(np.float32).max\n",
        "        self.max_weight = 4000000\n",
        "        self.min_weight = 0\n",
        "        self.df = pd.read_csv('mempool.csv')[['tx_id','fee','weight']]\n",
        "        self.low = np.array(\n",
        "            [self.min_fee, self.min_weight], dtype=np.float32\n",
        "        )\n",
        "        self.high = np.array(\n",
        "            [self.max_fee, self.max_weight], dtype=np.float32\n",
        "        )\n",
        "\n",
        "        self.action_space = spaces.Discrete(2)\n",
        "        self.observation_space = spaces.Box(\n",
        "            self.low, self.high, dtype=np.float32\n",
        "        )\n",
        "\n",
        "        self.seed()\n",
        "        self.current_fee = 0\n",
        "        self.current_weight = 0\n",
        "        self.c = 1\n",
        "        self.select_hash = []\n",
        "\n",
        "    def seed(self, seed=None):\n",
        "        self.np_random, seed = seeding.np_random(seed)\n",
        "        return [seed]\n",
        "\n",
        "    def select_transaction(self):\n",
        "        seed = np.random.choice(np.arange(len(self.df)), 1, replace=False)[0]\n",
        "        return self.df.values[seed]\n",
        "\n",
        "    def step(self, action):\n",
        "        assert self.action_space.contains(action), \"%r (%s) invalid\" % (action, type(action))\n",
        "\n",
        "        fee , weight  = self.state\n",
        "        select_hash , selected_fee , selected_weight = self.select_transaction()\n",
        "\n",
        "        if (fee + selected_fee) > (self.current_fee / self.c):\n",
        "          self.current_fee  = fee + selected_fee\n",
        "          self.current_weight = weight + selected_weight\n",
        "          self.c = self.c + 0.5\n",
        "          self.select_hash.append(select_hash)\n",
        "          reward = 1\n",
        "\n",
        "        elif (fee + selected_fee) <= (self.current_fee / self.c):\n",
        "          reward = -1\n",
        "\n",
        "\n",
        "        done = bool(\n",
        "          self.current_weight > self.max_weight\n",
        "        )\n",
        "\n",
        "        self.state = (self.current_fee, self.current_weight)\n",
        "        return np.array(self.state), reward, done, {}\n",
        "\n",
        "    def reset(self):\n",
        "        self.state = np.array([0,0])\n",
        "        self.current_fee, self.current_weight = 0 , 0\n",
        "        self.c = 1\n",
        "        self.select_hash = []\n",
        "        return np.array(self.state)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Writing env.py\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5aKZ0lB5Ut2k"
      },
      "source": [
        "env=FeesMaximizationEnv()"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zqbywNn_Uv2J",
        "outputId": "f19c3c71-1600-417f-c97f-5108abc58efa"
      },
      "source": [
        "episodes = 5\n",
        "for episode in range(1, episodes+1):\n",
        "    state = env.reset()\n",
        "    done = False\n",
        "    score = 0 \n",
        "    \n",
        "    while not done:\n",
        "        action = env.action_space.sample()\n",
        "        n_state, reward, done, info = env.step(action)\n",
        "        score+=reward\n",
        "    print(len(env.select_hash))\n",
        "    print('Episode:{} Score:{}'.format(episode, score))\n",
        "env.close()"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1972\n",
            "Episode:1 Score:1972\n",
            "2119\n",
            "Episode:2 Score:2119\n",
            "2172\n",
            "Episode:3 Score:2172\n",
            "2099\n",
            "Episode:4 Score:2099\n",
            "2057\n",
            "Episode:5 Score:2057\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7e_DfYoTVHNe",
        "outputId": "43f42188-f440-4a5f-c08c-ef277936792d"
      },
      "source": [
        "env = DummyVecEnv([lambda: env])\n",
        "model = PPO('MlpPolicy', env, verbose = 1)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using cpu device\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kR3lp4TNX6CQ",
        "outputId": "5dd871b0-1ea5-4e64-c4a0-6c47fba9edef"
      },
      "source": [
        "model.learn(total_timesteps=200000)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-----------------------------\n",
            "| time/              |      |\n",
            "|    fps             | 746  |\n",
            "|    iterations      | 1    |\n",
            "|    time_elapsed    | 2    |\n",
            "|    total_timesteps | 2048 |\n",
            "-----------------------------\n",
            "----------------------------------------\n",
            "| time/                   |            |\n",
            "|    fps                  | 638        |\n",
            "|    iterations           | 2          |\n",
            "|    time_elapsed         | 6          |\n",
            "|    total_timesteps      | 4096       |\n",
            "| train/                  |            |\n",
            "|    approx_kl            | 0.01267374 |\n",
            "|    clip_fraction        | 0.0598     |\n",
            "|    clip_range           | 0.2        |\n",
            "|    entropy_loss         | -0.68      |\n",
            "|    explained_variance   | 0.0505     |\n",
            "|    learning_rate        | 0.0003     |\n",
            "|    loss                 | 13.8       |\n",
            "|    n_updates            | 10         |\n",
            "|    policy_gradient_loss | -0.00202   |\n",
            "|    value_loss           | 98.9       |\n",
            "----------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 608          |\n",
            "|    iterations           | 3            |\n",
            "|    time_elapsed         | 10           |\n",
            "|    total_timesteps      | 6144         |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0031075478 |\n",
            "|    clip_fraction        | 0            |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.674       |\n",
            "|    explained_variance   | -0.0231      |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 33.6         |\n",
            "|    n_updates            | 20           |\n",
            "|    policy_gradient_loss | 0.0001       |\n",
            "|    value_loss           | 131          |\n",
            "------------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 596         |\n",
            "|    iterations           | 4           |\n",
            "|    time_elapsed         | 13          |\n",
            "|    total_timesteps      | 8192        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.016611116 |\n",
            "|    clip_fraction        | 0.154       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.691      |\n",
            "|    explained_variance   | -0.0144     |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 25.5        |\n",
            "|    n_updates            | 30          |\n",
            "|    policy_gradient_loss | -0.00488    |\n",
            "|    value_loss           | 111         |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 584         |\n",
            "|    iterations           | 5           |\n",
            "|    time_elapsed         | 17          |\n",
            "|    total_timesteps      | 10240       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.017734967 |\n",
            "|    clip_fraction        | 0.0853      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.675      |\n",
            "|    explained_variance   | -0.0434     |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 19.2        |\n",
            "|    n_updates            | 40          |\n",
            "|    policy_gradient_loss | -0.0049     |\n",
            "|    value_loss           | 91.3        |\n",
            "-----------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 582          |\n",
            "|    iterations           | 6            |\n",
            "|    time_elapsed         | 21           |\n",
            "|    total_timesteps      | 12288        |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0036372594 |\n",
            "|    clip_fraction        | 0.00903      |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.657       |\n",
            "|    explained_variance   | -0.0141      |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 13.2         |\n",
            "|    n_updates            | 50           |\n",
            "|    policy_gradient_loss | -0.000119    |\n",
            "|    value_loss           | 74.3         |\n",
            "------------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 581          |\n",
            "|    iterations           | 7            |\n",
            "|    time_elapsed         | 24           |\n",
            "|    total_timesteps      | 14336        |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0113299675 |\n",
            "|    clip_fraction        | 0.0507       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.653       |\n",
            "|    explained_variance   | -0.0104      |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 11.7         |\n",
            "|    n_updates            | 60           |\n",
            "|    policy_gradient_loss | -0.0012      |\n",
            "|    value_loss           | 60.1         |\n",
            "------------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 580          |\n",
            "|    iterations           | 8            |\n",
            "|    time_elapsed         | 28           |\n",
            "|    total_timesteps      | 16384        |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0118159475 |\n",
            "|    clip_fraction        | 0.0506       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.688       |\n",
            "|    explained_variance   | -0.00735     |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 17.2         |\n",
            "|    n_updates            | 70           |\n",
            "|    policy_gradient_loss | -0.00131     |\n",
            "|    value_loss           | 48.6         |\n",
            "------------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 577         |\n",
            "|    iterations           | 9           |\n",
            "|    time_elapsed         | 31          |\n",
            "|    total_timesteps      | 18432       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.007721114 |\n",
            "|    clip_fraction        | 0.0247      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.692      |\n",
            "|    explained_variance   | -0.00469    |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 6.94        |\n",
            "|    n_updates            | 80          |\n",
            "|    policy_gradient_loss | -0.00034    |\n",
            "|    value_loss           | 39.5        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 576         |\n",
            "|    iterations           | 10          |\n",
            "|    time_elapsed         | 35          |\n",
            "|    total_timesteps      | 20480       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.010744072 |\n",
            "|    clip_fraction        | 0.0433      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.665      |\n",
            "|    explained_variance   | -0.00296    |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 4.52        |\n",
            "|    n_updates            | 90          |\n",
            "|    policy_gradient_loss | -0.00394    |\n",
            "|    value_loss           | 32.9        |\n",
            "-----------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 575          |\n",
            "|    iterations           | 11           |\n",
            "|    time_elapsed         | 39           |\n",
            "|    total_timesteps      | 22528        |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0029580274 |\n",
            "|    clip_fraction        | 0.034        |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.646       |\n",
            "|    explained_variance   | -0.00181     |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 11.3         |\n",
            "|    n_updates            | 100          |\n",
            "|    policy_gradient_loss | -0.000762    |\n",
            "|    value_loss           | 29.2         |\n",
            "------------------------------------------\n",
            "----------------------------------------\n",
            "| time/                   |            |\n",
            "|    fps                  | 574        |\n",
            "|    iterations           | 12         |\n",
            "|    time_elapsed         | 42         |\n",
            "|    total_timesteps      | 24576      |\n",
            "| train/                  |            |\n",
            "|    approx_kl            | 0.01851387 |\n",
            "|    clip_fraction        | 0.0243     |\n",
            "|    clip_range           | 0.2        |\n",
            "|    entropy_loss         | -0.679     |\n",
            "|    explained_variance   | -0.000847  |\n",
            "|    learning_rate        | 0.0003     |\n",
            "|    loss                 | 12.4       |\n",
            "|    n_updates            | 110        |\n",
            "|    policy_gradient_loss | 0.000259   |\n",
            "|    value_loss           | 28         |\n",
            "----------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 573          |\n",
            "|    iterations           | 13           |\n",
            "|    time_elapsed         | 46           |\n",
            "|    total_timesteps      | 26624        |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0073934644 |\n",
            "|    clip_fraction        | 0.0622       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.687       |\n",
            "|    explained_variance   | -0.00206     |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 2.42         |\n",
            "|    n_updates            | 120          |\n",
            "|    policy_gradient_loss | -0.00186     |\n",
            "|    value_loss           | 27.9         |\n",
            "------------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 573         |\n",
            "|    iterations           | 14          |\n",
            "|    time_elapsed         | 50          |\n",
            "|    total_timesteps      | 28672       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.012761956 |\n",
            "|    clip_fraction        | 0.0836      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.683      |\n",
            "|    explained_variance   | -0.000272   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 14.5        |\n",
            "|    n_updates            | 130         |\n",
            "|    policy_gradient_loss | -0.00106    |\n",
            "|    value_loss           | 28.5        |\n",
            "-----------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 572          |\n",
            "|    iterations           | 15           |\n",
            "|    time_elapsed         | 53           |\n",
            "|    total_timesteps      | 30720        |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0038421033 |\n",
            "|    clip_fraction        | 0.0273       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.682       |\n",
            "|    explained_variance   | -6.45e-05    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 5.78         |\n",
            "|    n_updates            | 140          |\n",
            "|    policy_gradient_loss | 0.000301     |\n",
            "|    value_loss           | 29.7         |\n",
            "------------------------------------------\n",
            "----------------------------------------\n",
            "| time/                   |            |\n",
            "|    fps                  | 572        |\n",
            "|    iterations           | 16         |\n",
            "|    time_elapsed         | 57         |\n",
            "|    total_timesteps      | 32768      |\n",
            "| train/                  |            |\n",
            "|    approx_kl            | 0.00586396 |\n",
            "|    clip_fraction        | 0.101      |\n",
            "|    clip_range           | 0.2        |\n",
            "|    entropy_loss         | -0.691     |\n",
            "|    explained_variance   | -0.000102  |\n",
            "|    learning_rate        | 0.0003     |\n",
            "|    loss                 | 5.8        |\n",
            "|    n_updates            | 150        |\n",
            "|    policy_gradient_loss | -0.00106   |\n",
            "|    value_loss           | 30.8       |\n",
            "----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 572         |\n",
            "|    iterations           | 17          |\n",
            "|    time_elapsed         | 60          |\n",
            "|    total_timesteps      | 34816       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.013195218 |\n",
            "|    clip_fraction        | 0.032       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.681      |\n",
            "|    explained_variance   | -1.56e-05   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 0.0166      |\n",
            "|    n_updates            | 160         |\n",
            "|    policy_gradient_loss | -0.0015     |\n",
            "|    value_loss           | 0.867       |\n",
            "-----------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 572          |\n",
            "|    iterations           | 18           |\n",
            "|    time_elapsed         | 64           |\n",
            "|    total_timesteps      | 36864        |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0038518277 |\n",
            "|    clip_fraction        | 0.00562      |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.69        |\n",
            "|    explained_variance   | -0.000121    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 97.5         |\n",
            "|    n_updates            | 170          |\n",
            "|    policy_gradient_loss | 0.000291     |\n",
            "|    value_loss           | 64.8         |\n",
            "------------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 571         |\n",
            "|    iterations           | 19          |\n",
            "|    time_elapsed         | 68          |\n",
            "|    total_timesteps      | 38912       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.016222285 |\n",
            "|    clip_fraction        | 0.0608      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.668      |\n",
            "|    explained_variance   | -5.53e-05   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 1.01        |\n",
            "|    n_updates            | 180         |\n",
            "|    policy_gradient_loss | -0.00113    |\n",
            "|    value_loss           | 33.5        |\n",
            "-----------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 571          |\n",
            "|    iterations           | 20           |\n",
            "|    time_elapsed         | 71           |\n",
            "|    total_timesteps      | 40960        |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0070387796 |\n",
            "|    clip_fraction        | 0.0491       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.67        |\n",
            "|    explained_variance   | -4.71e-05    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 23.7         |\n",
            "|    n_updates            | 190          |\n",
            "|    policy_gradient_loss | -0.00162     |\n",
            "|    value_loss           | 34.3         |\n",
            "------------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 570         |\n",
            "|    iterations           | 21          |\n",
            "|    time_elapsed         | 75          |\n",
            "|    total_timesteps      | 43008       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.005544323 |\n",
            "|    clip_fraction        | 0.00918     |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.674      |\n",
            "|    explained_variance   | -3.5e-05    |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 9.27        |\n",
            "|    n_updates            | 200         |\n",
            "|    policy_gradient_loss | -0.000295   |\n",
            "|    value_loss           | 35          |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 570         |\n",
            "|    iterations           | 22          |\n",
            "|    time_elapsed         | 78          |\n",
            "|    total_timesteps      | 45056       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.008040026 |\n",
            "|    clip_fraction        | 0.0996      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.661      |\n",
            "|    explained_variance   | -2.78e-05   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 4.42        |\n",
            "|    n_updates            | 210         |\n",
            "|    policy_gradient_loss | -0.00158    |\n",
            "|    value_loss           | 35.5        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 570         |\n",
            "|    iterations           | 23          |\n",
            "|    time_elapsed         | 82          |\n",
            "|    total_timesteps      | 47104       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.011520429 |\n",
            "|    clip_fraction        | 0.0792      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.688      |\n",
            "|    explained_variance   | -2.32e-05   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 9.32        |\n",
            "|    n_updates            | 220         |\n",
            "|    policy_gradient_loss | -0.0024     |\n",
            "|    value_loss           | 36          |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 569         |\n",
            "|    iterations           | 24          |\n",
            "|    time_elapsed         | 86          |\n",
            "|    total_timesteps      | 49152       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.011626018 |\n",
            "|    clip_fraction        | 0.15        |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.674      |\n",
            "|    explained_variance   | -1.99e-05   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 0.912       |\n",
            "|    n_updates            | 230         |\n",
            "|    policy_gradient_loss | -0.00435    |\n",
            "|    value_loss           | 36.5        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 569         |\n",
            "|    iterations           | 25          |\n",
            "|    time_elapsed         | 89          |\n",
            "|    total_timesteps      | 51200       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.013548803 |\n",
            "|    clip_fraction        | 0.122       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.688      |\n",
            "|    explained_variance   | -1.79e-05   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 0.521       |\n",
            "|    n_updates            | 240         |\n",
            "|    policy_gradient_loss | -0.00364    |\n",
            "|    value_loss           | 36.8        |\n",
            "-----------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 569          |\n",
            "|    iterations           | 26           |\n",
            "|    time_elapsed         | 93           |\n",
            "|    total_timesteps      | 53248        |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0017473188 |\n",
            "|    clip_fraction        | 0            |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.686       |\n",
            "|    explained_variance   | -1.61e-05    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 13.9         |\n",
            "|    n_updates            | 250          |\n",
            "|    policy_gradient_loss | -0.00014     |\n",
            "|    value_loss           | 37.1         |\n",
            "------------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 569          |\n",
            "|    iterations           | 27           |\n",
            "|    time_elapsed         | 97           |\n",
            "|    total_timesteps      | 55296        |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0024143336 |\n",
            "|    clip_fraction        | 0.0531       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.691       |\n",
            "|    explained_variance   | -1.3e-05     |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 58.6         |\n",
            "|    n_updates            | 260          |\n",
            "|    policy_gradient_loss | -0.00264     |\n",
            "|    value_loss           | 37.3         |\n",
            "------------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 569         |\n",
            "|    iterations           | 28          |\n",
            "|    time_elapsed         | 100         |\n",
            "|    total_timesteps      | 57344       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.007904166 |\n",
            "|    clip_fraction        | 0           |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.688      |\n",
            "|    explained_variance   | -1.18e-05   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 0.148       |\n",
            "|    n_updates            | 270         |\n",
            "|    policy_gradient_loss | -5e-05      |\n",
            "|    value_loss           | 37.5        |\n",
            "-----------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 569          |\n",
            "|    iterations           | 29           |\n",
            "|    time_elapsed         | 104          |\n",
            "|    total_timesteps      | 59392        |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0012282792 |\n",
            "|    clip_fraction        | 0            |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.689       |\n",
            "|    explained_variance   | -8.7e-06     |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 0.288        |\n",
            "|    n_updates            | 280          |\n",
            "|    policy_gradient_loss | -0.000182    |\n",
            "|    value_loss           | 37.8         |\n",
            "------------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 570         |\n",
            "|    iterations           | 30          |\n",
            "|    time_elapsed         | 107         |\n",
            "|    total_timesteps      | 61440       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.023370579 |\n",
            "|    clip_fraction        | 0.172       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.687      |\n",
            "|    explained_variance   | -8.46e-06   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 0.707       |\n",
            "|    n_updates            | 290         |\n",
            "|    policy_gradient_loss | -0.00402    |\n",
            "|    value_loss           | 37.8        |\n",
            "-----------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 570          |\n",
            "|    iterations           | 31           |\n",
            "|    time_elapsed         | 111          |\n",
            "|    total_timesteps      | 63488        |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0145391505 |\n",
            "|    clip_fraction        | 0.0757       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.661       |\n",
            "|    explained_variance   | -7.51e-06    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 20.6         |\n",
            "|    n_updates            | 300          |\n",
            "|    policy_gradient_loss | -0.00489     |\n",
            "|    value_loss           | 37.9         |\n",
            "------------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 569         |\n",
            "|    iterations           | 32          |\n",
            "|    time_elapsed         | 115         |\n",
            "|    total_timesteps      | 65536       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.014956092 |\n",
            "|    clip_fraction        | 0.0298      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.671      |\n",
            "|    explained_variance   | -5.72e-06   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 51.7        |\n",
            "|    n_updates            | 310         |\n",
            "|    policy_gradient_loss | 0.000141    |\n",
            "|    value_loss           | 38          |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 570         |\n",
            "|    iterations           | 33          |\n",
            "|    time_elapsed         | 118         |\n",
            "|    total_timesteps      | 67584       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.015376268 |\n",
            "|    clip_fraction        | 0.0614      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.69       |\n",
            "|    explained_variance   | -5.6e-06    |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 0.0609      |\n",
            "|    n_updates            | 320         |\n",
            "|    policy_gradient_loss | -0.00143    |\n",
            "|    value_loss           | 38.1        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 570         |\n",
            "|    iterations           | 34          |\n",
            "|    time_elapsed         | 122         |\n",
            "|    total_timesteps      | 69632       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.005130024 |\n",
            "|    clip_fraction        | 0.0165      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.688      |\n",
            "|    explained_variance   | -5.13e-06   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 3.73        |\n",
            "|    n_updates            | 330         |\n",
            "|    policy_gradient_loss | -0.00105    |\n",
            "|    value_loss           | 38.2        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 570         |\n",
            "|    iterations           | 35          |\n",
            "|    time_elapsed         | 125         |\n",
            "|    total_timesteps      | 71680       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.010847207 |\n",
            "|    clip_fraction        | 0           |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.689      |\n",
            "|    explained_variance   | -4.29e-06   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 75.5        |\n",
            "|    n_updates            | 340         |\n",
            "|    policy_gradient_loss | -3.11e-05   |\n",
            "|    value_loss           | 38.2        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 571         |\n",
            "|    iterations           | 36          |\n",
            "|    time_elapsed         | 129         |\n",
            "|    total_timesteps      | 73728       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.020482391 |\n",
            "|    clip_fraction        | 0.046       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.675      |\n",
            "|    explained_variance   | -4.17e-06   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 0.265       |\n",
            "|    n_updates            | 350         |\n",
            "|    policy_gradient_loss | -0.00435    |\n",
            "|    value_loss           | 38.3        |\n",
            "-----------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 571          |\n",
            "|    iterations           | 37           |\n",
            "|    time_elapsed         | 132          |\n",
            "|    total_timesteps      | 75776        |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0019332375 |\n",
            "|    clip_fraction        | 0.0154       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.663       |\n",
            "|    explained_variance   | -3.22e-06    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 55.4         |\n",
            "|    n_updates            | 360          |\n",
            "|    policy_gradient_loss | 0.000474     |\n",
            "|    value_loss           | 38.4         |\n",
            "------------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 571         |\n",
            "|    iterations           | 38          |\n",
            "|    time_elapsed         | 136         |\n",
            "|    total_timesteps      | 77824       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.018682176 |\n",
            "|    clip_fraction        | 0.0659      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.643      |\n",
            "|    explained_variance   | -2.98e-06   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 5.38        |\n",
            "|    n_updates            | 370         |\n",
            "|    policy_gradient_loss | -0.00089    |\n",
            "|    value_loss           | 38.4        |\n",
            "-----------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 571          |\n",
            "|    iterations           | 39           |\n",
            "|    time_elapsed         | 139          |\n",
            "|    total_timesteps      | 79872        |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0031145336 |\n",
            "|    clip_fraction        | 0.0874       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.593       |\n",
            "|    explained_variance   | -2.5e-06     |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 63.2         |\n",
            "|    n_updates            | 380          |\n",
            "|    policy_gradient_loss | -0.00334     |\n",
            "|    value_loss           | 38.4         |\n",
            "------------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 572          |\n",
            "|    iterations           | 40           |\n",
            "|    time_elapsed         | 143          |\n",
            "|    total_timesteps      | 81920        |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0017670032 |\n",
            "|    clip_fraction        | 0.113        |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.623       |\n",
            "|    explained_variance   | -2.03e-06    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 14.4         |\n",
            "|    n_updates            | 390          |\n",
            "|    policy_gradient_loss | -0.00254     |\n",
            "|    value_loss           | 38.4         |\n",
            "------------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 572         |\n",
            "|    iterations           | 41          |\n",
            "|    time_elapsed         | 146         |\n",
            "|    total_timesteps      | 83968       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.006136464 |\n",
            "|    clip_fraction        | 0.078       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.562      |\n",
            "|    explained_variance   | -2.15e-06   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 0.364       |\n",
            "|    n_updates            | 400         |\n",
            "|    policy_gradient_loss | -0.00191    |\n",
            "|    value_loss           | 38.5        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 572         |\n",
            "|    iterations           | 42          |\n",
            "|    time_elapsed         | 150         |\n",
            "|    total_timesteps      | 86016       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.007739642 |\n",
            "|    clip_fraction        | 0.0411      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.587      |\n",
            "|    explained_variance   | -1.67e-06   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 13.8        |\n",
            "|    n_updates            | 410         |\n",
            "|    policy_gradient_loss | -0.000138   |\n",
            "|    value_loss           | 38.5        |\n",
            "-----------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 572          |\n",
            "|    iterations           | 43           |\n",
            "|    time_elapsed         | 153          |\n",
            "|    total_timesteps      | 88064        |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0030019407 |\n",
            "|    clip_fraction        | 0.0794       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.556       |\n",
            "|    explained_variance   | -1.43e-06    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 0.611        |\n",
            "|    n_updates            | 420          |\n",
            "|    policy_gradient_loss | -0.00162     |\n",
            "|    value_loss           | 38.5         |\n",
            "------------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 572          |\n",
            "|    iterations           | 44           |\n",
            "|    time_elapsed         | 157          |\n",
            "|    total_timesteps      | 90112        |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0107588805 |\n",
            "|    clip_fraction        | 0.0541       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.616       |\n",
            "|    explained_variance   | -1.55e-06    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 0.226        |\n",
            "|    n_updates            | 430          |\n",
            "|    policy_gradient_loss | -0.000319    |\n",
            "|    value_loss           | 38.6         |\n",
            "------------------------------------------\n",
            "----------------------------------------\n",
            "| time/                   |            |\n",
            "|    fps                  | 572        |\n",
            "|    iterations           | 45         |\n",
            "|    time_elapsed         | 160        |\n",
            "|    total_timesteps      | 92160      |\n",
            "| train/                  |            |\n",
            "|    approx_kl            | 0.01368201 |\n",
            "|    clip_fraction        | 0.0384     |\n",
            "|    clip_range           | 0.2        |\n",
            "|    entropy_loss         | -0.585     |\n",
            "|    explained_variance   | -1.43e-06  |\n",
            "|    learning_rate        | 0.0003     |\n",
            "|    loss                 | 7.9        |\n",
            "|    n_updates            | 440        |\n",
            "|    policy_gradient_loss | -0.00283   |\n",
            "|    value_loss           | 38.5       |\n",
            "----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 572         |\n",
            "|    iterations           | 46          |\n",
            "|    time_elapsed         | 164         |\n",
            "|    total_timesteps      | 94208       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.008492254 |\n",
            "|    clip_fraction        | 0.0701      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.598      |\n",
            "|    explained_variance   | -1.19e-06   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 6.1         |\n",
            "|    n_updates            | 450         |\n",
            "|    policy_gradient_loss | 0.000453    |\n",
            "|    value_loss           | 38.5        |\n",
            "-----------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 573          |\n",
            "|    iterations           | 47           |\n",
            "|    time_elapsed         | 167          |\n",
            "|    total_timesteps      | 96256        |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0059088934 |\n",
            "|    clip_fraction        | 0.0556       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.651       |\n",
            "|    explained_variance   | -1.07e-06    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 16.1         |\n",
            "|    n_updates            | 460          |\n",
            "|    policy_gradient_loss | 0.00018      |\n",
            "|    value_loss           | 38.5         |\n",
            "------------------------------------------\n",
            "-------------------------------------------\n",
            "| time/                   |               |\n",
            "|    fps                  | 573           |\n",
            "|    iterations           | 48            |\n",
            "|    time_elapsed         | 171           |\n",
            "|    total_timesteps      | 98304         |\n",
            "| train/                  |               |\n",
            "|    approx_kl            | 1.3915938e-05 |\n",
            "|    clip_fraction        | 0             |\n",
            "|    clip_range           | 0.2           |\n",
            "|    entropy_loss         | -0.642        |\n",
            "|    explained_variance   | -8.34e-07     |\n",
            "|    learning_rate        | 0.0003        |\n",
            "|    loss                 | 43.7          |\n",
            "|    n_updates            | 470           |\n",
            "|    policy_gradient_loss | 0.00025       |\n",
            "|    value_loss           | 38.5          |\n",
            "-------------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 573          |\n",
            "|    iterations           | 49           |\n",
            "|    time_elapsed         | 175          |\n",
            "|    total_timesteps      | 100352       |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0017300509 |\n",
            "|    clip_fraction        | 0            |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.615       |\n",
            "|    explained_variance   | -1.07e-06    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 0.744        |\n",
            "|    n_updates            | 480          |\n",
            "|    policy_gradient_loss | 0.000393     |\n",
            "|    value_loss           | 38.5         |\n",
            "------------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 573          |\n",
            "|    iterations           | 50           |\n",
            "|    time_elapsed         | 178          |\n",
            "|    total_timesteps      | 102400       |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0076110307 |\n",
            "|    clip_fraction        | 0.0381       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.625       |\n",
            "|    explained_variance   | -8.34e-07    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 0.217        |\n",
            "|    n_updates            | 490          |\n",
            "|    policy_gradient_loss | 0.000161     |\n",
            "|    value_loss           | 38.5         |\n",
            "------------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 573         |\n",
            "|    iterations           | 51          |\n",
            "|    time_elapsed         | 182         |\n",
            "|    total_timesteps      | 104448      |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.014702217 |\n",
            "|    clip_fraction        | 0.069       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.592      |\n",
            "|    explained_variance   | -8.34e-07   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 6.08        |\n",
            "|    n_updates            | 500         |\n",
            "|    policy_gradient_loss | -0.00562    |\n",
            "|    value_loss           | 38.5        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 573         |\n",
            "|    iterations           | 52          |\n",
            "|    time_elapsed         | 185         |\n",
            "|    total_timesteps      | 106496      |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.005028784 |\n",
            "|    clip_fraction        | 0.0813      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.616      |\n",
            "|    explained_variance   | -8.34e-07   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 0.148       |\n",
            "|    n_updates            | 510         |\n",
            "|    policy_gradient_loss | -0.00147    |\n",
            "|    value_loss           | 38.5        |\n",
            "-----------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 573          |\n",
            "|    iterations           | 53           |\n",
            "|    time_elapsed         | 189          |\n",
            "|    total_timesteps      | 108544       |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0037898046 |\n",
            "|    clip_fraction        | 0.00303      |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.6         |\n",
            "|    explained_variance   | -8.34e-07    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 12.1         |\n",
            "|    n_updates            | 520          |\n",
            "|    policy_gradient_loss | -0.000625    |\n",
            "|    value_loss           | 38.5         |\n",
            "------------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 573         |\n",
            "|    iterations           | 54          |\n",
            "|    time_elapsed         | 192         |\n",
            "|    total_timesteps      | 110592      |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.010569524 |\n",
            "|    clip_fraction        | 0.0423      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.544      |\n",
            "|    explained_variance   | -8.34e-07   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 30.7        |\n",
            "|    n_updates            | 530         |\n",
            "|    policy_gradient_loss | -0.00217    |\n",
            "|    value_loss           | 38.5        |\n",
            "-----------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 573          |\n",
            "|    iterations           | 55           |\n",
            "|    time_elapsed         | 196          |\n",
            "|    total_timesteps      | 112640       |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0005937819 |\n",
            "|    clip_fraction        | 0.000293     |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.553       |\n",
            "|    explained_variance   | -9.54e-07    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 52.4         |\n",
            "|    n_updates            | 540          |\n",
            "|    policy_gradient_loss | 0.000248     |\n",
            "|    value_loss           | 75.5         |\n",
            "------------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 574          |\n",
            "|    iterations           | 56           |\n",
            "|    time_elapsed         | 199          |\n",
            "|    total_timesteps      | 114688       |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0029593692 |\n",
            "|    clip_fraction        | 0.0961       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.602       |\n",
            "|    explained_variance   | 5.96e-08     |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 0.0243       |\n",
            "|    n_updates            | 550          |\n",
            "|    policy_gradient_loss | -0.000988    |\n",
            "|    value_loss           | 0.0285       |\n",
            "------------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 574         |\n",
            "|    iterations           | 57          |\n",
            "|    time_elapsed         | 203         |\n",
            "|    total_timesteps      | 116736      |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.011116966 |\n",
            "|    clip_fraction        | 0.0101      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.625      |\n",
            "|    explained_variance   | -4.77e-07   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 65.3        |\n",
            "|    n_updates            | 560         |\n",
            "|    policy_gradient_loss | 0.000598    |\n",
            "|    value_loss           | 38.4        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 574         |\n",
            "|    iterations           | 58          |\n",
            "|    time_elapsed         | 206         |\n",
            "|    total_timesteps      | 118784      |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.004098974 |\n",
            "|    clip_fraction        | 0.0658      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.675      |\n",
            "|    explained_variance   | -4.77e-07   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 0.165       |\n",
            "|    n_updates            | 570         |\n",
            "|    policy_gradient_loss | -0.00126    |\n",
            "|    value_loss           | 38.4        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 574         |\n",
            "|    iterations           | 59          |\n",
            "|    time_elapsed         | 210         |\n",
            "|    total_timesteps      | 120832      |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.007856131 |\n",
            "|    clip_fraction        | 0           |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.691      |\n",
            "|    explained_variance   | -2.38e-07   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 56.4        |\n",
            "|    n_updates            | 580         |\n",
            "|    policy_gradient_loss | 0.00098     |\n",
            "|    value_loss           | 35.9        |\n",
            "-----------------------------------------\n",
            "----------------------------------------\n",
            "| time/                   |            |\n",
            "|    fps                  | 574        |\n",
            "|    iterations           | 60         |\n",
            "|    time_elapsed         | 213        |\n",
            "|    total_timesteps      | 122880     |\n",
            "| train/                  |            |\n",
            "|    approx_kl            | 0.01908062 |\n",
            "|    clip_fraction        | 0.0537     |\n",
            "|    clip_range           | 0.2        |\n",
            "|    entropy_loss         | -0.686     |\n",
            "|    explained_variance   | -9.54e-07  |\n",
            "|    learning_rate        | 0.0003     |\n",
            "|    loss                 | 120        |\n",
            "|    n_updates            | 590        |\n",
            "|    policy_gradient_loss | -0.0015    |\n",
            "|    value_loss           | 76.5       |\n",
            "----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 574         |\n",
            "|    iterations           | 61          |\n",
            "|    time_elapsed         | 217         |\n",
            "|    total_timesteps      | 124928      |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.010133952 |\n",
            "|    clip_fraction        | 0.0786      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.69       |\n",
            "|    explained_variance   | -2.38e-07   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 0.157       |\n",
            "|    n_updates            | 600         |\n",
            "|    policy_gradient_loss | -0.00199    |\n",
            "|    value_loss           | 38.3        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 574         |\n",
            "|    iterations           | 62          |\n",
            "|    time_elapsed         | 220         |\n",
            "|    total_timesteps      | 126976      |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.014650067 |\n",
            "|    clip_fraction        | 0.0463      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.686      |\n",
            "|    explained_variance   | 0           |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | -0.0221     |\n",
            "|    n_updates            | 610         |\n",
            "|    policy_gradient_loss | -0.000513   |\n",
            "|    value_loss           | 0.0213      |\n",
            "-----------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 574          |\n",
            "|    iterations           | 63           |\n",
            "|    time_elapsed         | 224          |\n",
            "|    total_timesteps      | 129024       |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0151080135 |\n",
            "|    clip_fraction        | 0.0896       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.643       |\n",
            "|    explained_variance   | -2.38e-07    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 6.6          |\n",
            "|    n_updates            | 620          |\n",
            "|    policy_gradient_loss | -0.00289     |\n",
            "|    value_loss           | 38.7         |\n",
            "------------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 574          |\n",
            "|    iterations           | 64           |\n",
            "|    time_elapsed         | 228          |\n",
            "|    total_timesteps      | 131072       |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0047128545 |\n",
            "|    clip_fraction        | 0.0214       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.618       |\n",
            "|    explained_variance   | -4.77e-07    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 0.345        |\n",
            "|    n_updates            | 630          |\n",
            "|    policy_gradient_loss | -0.000692    |\n",
            "|    value_loss           | 38.6         |\n",
            "------------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 574         |\n",
            "|    iterations           | 65          |\n",
            "|    time_elapsed         | 231         |\n",
            "|    total_timesteps      | 133120      |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.007481009 |\n",
            "|    clip_fraction        | 0.0533      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.629      |\n",
            "|    explained_variance   | -4.77e-07   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 21.6        |\n",
            "|    n_updates            | 640         |\n",
            "|    policy_gradient_loss | -0.000272   |\n",
            "|    value_loss           | 38.6        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 574         |\n",
            "|    iterations           | 66          |\n",
            "|    time_elapsed         | 235         |\n",
            "|    total_timesteps      | 135168      |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.014006955 |\n",
            "|    clip_fraction        | 0.057       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.59       |\n",
            "|    explained_variance   | -3.58e-07   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 1.95        |\n",
            "|    n_updates            | 650         |\n",
            "|    policy_gradient_loss | -0.00109    |\n",
            "|    value_loss           | 38.6        |\n",
            "-----------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 574          |\n",
            "|    iterations           | 67           |\n",
            "|    time_elapsed         | 238          |\n",
            "|    total_timesteps      | 137216       |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0065960065 |\n",
            "|    clip_fraction        | 0.0418       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.559       |\n",
            "|    explained_variance   | -9.54e-07    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 15.9         |\n",
            "|    n_updates            | 660          |\n",
            "|    policy_gradient_loss | -0.00125     |\n",
            "|    value_loss           | 75.9         |\n",
            "------------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 574         |\n",
            "|    iterations           | 68          |\n",
            "|    time_elapsed         | 242         |\n",
            "|    total_timesteps      | 139264      |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.004938867 |\n",
            "|    clip_fraction        | 0.0853      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.58       |\n",
            "|    explained_variance   | -2.38e-07   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 70.7        |\n",
            "|    n_updates            | 670         |\n",
            "|    policy_gradient_loss | -0.00146    |\n",
            "|    value_loss           | 38.1        |\n",
            "-----------------------------------------\n",
            "-------------------------------------------\n",
            "| time/                   |               |\n",
            "|    fps                  | 574           |\n",
            "|    iterations           | 69            |\n",
            "|    time_elapsed         | 246           |\n",
            "|    total_timesteps      | 141312        |\n",
            "| train/                  |               |\n",
            "|    approx_kl            | 0.00043453457 |\n",
            "|    clip_fraction        | 0.0282        |\n",
            "|    clip_range           | 0.2           |\n",
            "|    entropy_loss         | -0.559        |\n",
            "|    explained_variance   | -4.77e-07     |\n",
            "|    learning_rate        | 0.0003        |\n",
            "|    loss                 | 127           |\n",
            "|    n_updates            | 680           |\n",
            "|    policy_gradient_loss | -0.00226      |\n",
            "|    value_loss           | 38.1          |\n",
            "-------------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 573          |\n",
            "|    iterations           | 70           |\n",
            "|    time_elapsed         | 249          |\n",
            "|    total_timesteps      | 143360       |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0035450268 |\n",
            "|    clip_fraction        | 0.113        |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.527       |\n",
            "|    explained_variance   | -1.19e-07    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 82.9         |\n",
            "|    n_updates            | 690          |\n",
            "|    policy_gradient_loss | -0.0014      |\n",
            "|    value_loss           | 38.2         |\n",
            "------------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 574          |\n",
            "|    iterations           | 71           |\n",
            "|    time_elapsed         | 253          |\n",
            "|    total_timesteps      | 145408       |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0041089784 |\n",
            "|    clip_fraction        | 0.0201       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.52        |\n",
            "|    explained_variance   | -2.38e-07    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 5.42         |\n",
            "|    n_updates            | 700          |\n",
            "|    policy_gradient_loss | 0.000397     |\n",
            "|    value_loss           | 38.3         |\n",
            "------------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 573          |\n",
            "|    iterations           | 72           |\n",
            "|    time_elapsed         | 257          |\n",
            "|    total_timesteps      | 147456       |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0034669098 |\n",
            "|    clip_fraction        | 0.0459       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.495       |\n",
            "|    explained_variance   | -2.38e-07    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 9.35         |\n",
            "|    n_updates            | 710          |\n",
            "|    policy_gradient_loss | -0.00114     |\n",
            "|    value_loss           | 38.4         |\n",
            "------------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 573          |\n",
            "|    iterations           | 73           |\n",
            "|    time_elapsed         | 260          |\n",
            "|    total_timesteps      | 149504       |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0016027123 |\n",
            "|    clip_fraction        | 0.0129       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.527       |\n",
            "|    explained_variance   | -9.54e-07    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 4.98         |\n",
            "|    n_updates            | 720          |\n",
            "|    policy_gradient_loss | 9.38e-05     |\n",
            "|    value_loss           | 38.4         |\n",
            "------------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 573         |\n",
            "|    iterations           | 74          |\n",
            "|    time_elapsed         | 264         |\n",
            "|    total_timesteps      | 151552      |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.009016579 |\n",
            "|    clip_fraction        | 0.087       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.577      |\n",
            "|    explained_variance   | -1.19e-07   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 28.6        |\n",
            "|    n_updates            | 730         |\n",
            "|    policy_gradient_loss | -0.000653   |\n",
            "|    value_loss           | 38.5        |\n",
            "-----------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 573          |\n",
            "|    iterations           | 75           |\n",
            "|    time_elapsed         | 267          |\n",
            "|    total_timesteps      | 153600       |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0029716685 |\n",
            "|    clip_fraction        | 0.116        |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.621       |\n",
            "|    explained_variance   | -2.38e-07    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 11.4         |\n",
            "|    n_updates            | 740          |\n",
            "|    policy_gradient_loss | -0.00363     |\n",
            "|    value_loss           | 38.5         |\n",
            "------------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 573         |\n",
            "|    iterations           | 76          |\n",
            "|    time_elapsed         | 271         |\n",
            "|    total_timesteps      | 155648      |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.004006025 |\n",
            "|    clip_fraction        | 0.0112      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.594      |\n",
            "|    explained_variance   | -2.38e-07   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 46.1        |\n",
            "|    n_updates            | 750         |\n",
            "|    policy_gradient_loss | 0.000695    |\n",
            "|    value_loss           | 38.5        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 573         |\n",
            "|    iterations           | 77          |\n",
            "|    time_elapsed         | 274         |\n",
            "|    total_timesteps      | 157696      |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.005723696 |\n",
            "|    clip_fraction        | 0.086       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.57       |\n",
            "|    explained_variance   | -2.38e-07   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 0.195       |\n",
            "|    n_updates            | 760         |\n",
            "|    policy_gradient_loss | -0.0025     |\n",
            "|    value_loss           | 38.5        |\n",
            "-----------------------------------------\n",
            "-------------------------------------------\n",
            "| time/                   |               |\n",
            "|    fps                  | 574           |\n",
            "|    iterations           | 78            |\n",
            "|    time_elapsed         | 278           |\n",
            "|    total_timesteps      | 159744        |\n",
            "| train/                  |               |\n",
            "|    approx_kl            | 2.8155671e-05 |\n",
            "|    clip_fraction        | 0             |\n",
            "|    clip_range           | 0.2           |\n",
            "|    entropy_loss         | -0.587        |\n",
            "|    explained_variance   | -1.19e-07     |\n",
            "|    learning_rate        | 0.0003        |\n",
            "|    loss                 | 0.159         |\n",
            "|    n_updates            | 770           |\n",
            "|    policy_gradient_loss | 0.000312      |\n",
            "|    value_loss           | 38.4          |\n",
            "-------------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 574         |\n",
            "|    iterations           | 79          |\n",
            "|    time_elapsed         | 281         |\n",
            "|    total_timesteps      | 161792      |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.005172735 |\n",
            "|    clip_fraction        | 0.124       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.517      |\n",
            "|    explained_variance   | -1.19e-07   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 0.199       |\n",
            "|    n_updates            | 780         |\n",
            "|    policy_gradient_loss | -0.00188    |\n",
            "|    value_loss           | 38.5        |\n",
            "-----------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 574          |\n",
            "|    iterations           | 80           |\n",
            "|    time_elapsed         | 285          |\n",
            "|    total_timesteps      | 163840       |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0038136505 |\n",
            "|    clip_fraction        | 0.0641       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.488       |\n",
            "|    explained_variance   | -2.38e-07    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 0.162        |\n",
            "|    n_updates            | 790          |\n",
            "|    policy_gradient_loss | -0.00605     |\n",
            "|    value_loss           | 38.4         |\n",
            "------------------------------------------\n",
            "-------------------------------------------\n",
            "| time/                   |               |\n",
            "|    fps                  | 574           |\n",
            "|    iterations           | 81            |\n",
            "|    time_elapsed         | 288           |\n",
            "|    total_timesteps      | 165888        |\n",
            "| train/                  |               |\n",
            "|    approx_kl            | 0.00072854845 |\n",
            "|    clip_fraction        | 0.0135        |\n",
            "|    clip_range           | 0.2           |\n",
            "|    entropy_loss         | -0.46         |\n",
            "|    explained_variance   | -3.58e-07     |\n",
            "|    learning_rate        | 0.0003        |\n",
            "|    loss                 | 44.2          |\n",
            "|    n_updates            | 800           |\n",
            "|    policy_gradient_loss | -2.11e-05     |\n",
            "|    value_loss           | 38.5          |\n",
            "-------------------------------------------\n",
            "-------------------------------------------\n",
            "| time/                   |               |\n",
            "|    fps                  | 574           |\n",
            "|    iterations           | 82            |\n",
            "|    time_elapsed         | 292           |\n",
            "|    total_timesteps      | 167936        |\n",
            "| train/                  |               |\n",
            "|    approx_kl            | -0.0010509056 |\n",
            "|    clip_fraction        | 0             |\n",
            "|    clip_range           | 0.2           |\n",
            "|    entropy_loss         | -0.508        |\n",
            "|    explained_variance   | -3.58e-07     |\n",
            "|    learning_rate        | 0.0003        |\n",
            "|    loss                 | 5.41          |\n",
            "|    n_updates            | 810           |\n",
            "|    policy_gradient_loss | 0.000341      |\n",
            "|    value_loss           | 38.4          |\n",
            "-------------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 574          |\n",
            "|    iterations           | 83           |\n",
            "|    time_elapsed         | 296          |\n",
            "|    total_timesteps      | 169984       |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0011867762 |\n",
            "|    clip_fraction        | 0.0578       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.511       |\n",
            "|    explained_variance   | 1.79e-07     |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | -0.00252     |\n",
            "|    n_updates            | 820          |\n",
            "|    policy_gradient_loss | 0.00078      |\n",
            "|    value_loss           | 0.0153       |\n",
            "------------------------------------------\n",
            "-------------------------------------------\n",
            "| time/                   |               |\n",
            "|    fps                  | 574           |\n",
            "|    iterations           | 84            |\n",
            "|    time_elapsed         | 299           |\n",
            "|    total_timesteps      | 172032        |\n",
            "| train/                  |               |\n",
            "|    approx_kl            | 0.00085271365 |\n",
            "|    clip_fraction        | 0.0154        |\n",
            "|    clip_range           | 0.2           |\n",
            "|    entropy_loss         | -0.529        |\n",
            "|    explained_variance   | -3.58e-07     |\n",
            "|    learning_rate        | 0.0003        |\n",
            "|    loss                 | 13.3          |\n",
            "|    n_updates            | 830           |\n",
            "|    policy_gradient_loss | -0.000793     |\n",
            "|    value_loss           | 76.6          |\n",
            "-------------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 574          |\n",
            "|    iterations           | 85           |\n",
            "|    time_elapsed         | 303          |\n",
            "|    total_timesteps      | 174080       |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0010616465 |\n",
            "|    clip_fraction        | 0.00371      |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.557       |\n",
            "|    explained_variance   | -2.38e-07    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 0.354        |\n",
            "|    n_updates            | 840          |\n",
            "|    policy_gradient_loss | 0.000316     |\n",
            "|    value_loss           | 38.2         |\n",
            "------------------------------------------\n",
            "--------------------------------------------\n",
            "| time/                   |                |\n",
            "|    fps                  | 574            |\n",
            "|    iterations           | 86             |\n",
            "|    time_elapsed         | 306            |\n",
            "|    total_timesteps      | 176128         |\n",
            "| train/                  |                |\n",
            "|    approx_kl            | -0.00052745943 |\n",
            "|    clip_fraction        | 0              |\n",
            "|    clip_range           | 0.2            |\n",
            "|    entropy_loss         | -0.6           |\n",
            "|    explained_variance   | -3.58e-07      |\n",
            "|    learning_rate        | 0.0003         |\n",
            "|    loss                 | 43.4           |\n",
            "|    n_updates            | 850            |\n",
            "|    policy_gradient_loss | -6.18e-05      |\n",
            "|    value_loss           | 38.2           |\n",
            "--------------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 574          |\n",
            "|    iterations           | 87           |\n",
            "|    time_elapsed         | 310          |\n",
            "|    total_timesteps      | 178176       |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0026077284 |\n",
            "|    clip_fraction        | 0.0571       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.608       |\n",
            "|    explained_variance   | -3.58e-07    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 0.224        |\n",
            "|    n_updates            | 860          |\n",
            "|    policy_gradient_loss | -0.000513    |\n",
            "|    value_loss           | 38.4         |\n",
            "------------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 574          |\n",
            "|    iterations           | 88           |\n",
            "|    time_elapsed         | 313          |\n",
            "|    total_timesteps      | 180224       |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0038323752 |\n",
            "|    clip_fraction        | 0.0999       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.65        |\n",
            "|    explained_variance   | -1.19e-07    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 2.61         |\n",
            "|    n_updates            | 870          |\n",
            "|    policy_gradient_loss | 6.23e-05     |\n",
            "|    value_loss           | 38.4         |\n",
            "------------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 574          |\n",
            "|    iterations           | 89           |\n",
            "|    time_elapsed         | 317          |\n",
            "|    total_timesteps      | 182272       |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0056167278 |\n",
            "|    clip_fraction        | 0.17         |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.668       |\n",
            "|    explained_variance   | -1.19e-07    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 20.1         |\n",
            "|    n_updates            | 880          |\n",
            "|    policy_gradient_loss | -0.00326     |\n",
            "|    value_loss           | 38.4         |\n",
            "------------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 574          |\n",
            "|    iterations           | 90           |\n",
            "|    time_elapsed         | 320          |\n",
            "|    total_timesteps      | 184320       |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0062706335 |\n",
            "|    clip_fraction        | 0.0275       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.645       |\n",
            "|    explained_variance   | -2.38e-07    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 35.8         |\n",
            "|    n_updates            | 890          |\n",
            "|    policy_gradient_loss | -0.00267     |\n",
            "|    value_loss           | 38.4         |\n",
            "------------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 574          |\n",
            "|    iterations           | 91           |\n",
            "|    time_elapsed         | 324          |\n",
            "|    total_timesteps      | 186368       |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | -0.000167361 |\n",
            "|    clip_fraction        | 0            |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.632       |\n",
            "|    explained_variance   | -1.19e-07    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 48.5         |\n",
            "|    n_updates            | 900          |\n",
            "|    policy_gradient_loss | -0.000143    |\n",
            "|    value_loss           | 38.4         |\n",
            "------------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 574         |\n",
            "|    iterations           | 92          |\n",
            "|    time_elapsed         | 327         |\n",
            "|    total_timesteps      | 188416      |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.003184558 |\n",
            "|    clip_fraction        | 0           |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.6        |\n",
            "|    explained_variance   | -2.38e-07   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 25.7        |\n",
            "|    n_updates            | 910         |\n",
            "|    policy_gradient_loss | 2.38e-06    |\n",
            "|    value_loss           | 38.5        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 574         |\n",
            "|    iterations           | 93          |\n",
            "|    time_elapsed         | 331         |\n",
            "|    total_timesteps      | 190464      |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.002032441 |\n",
            "|    clip_fraction        | 0.0161      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.602      |\n",
            "|    explained_variance   | -3.58e-07   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 34.1        |\n",
            "|    n_updates            | 920         |\n",
            "|    policy_gradient_loss | 0.00023     |\n",
            "|    value_loss           | 38.4        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 574         |\n",
            "|    iterations           | 94          |\n",
            "|    time_elapsed         | 335         |\n",
            "|    total_timesteps      | 192512      |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.009896738 |\n",
            "|    clip_fraction        | 0.0971      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.647      |\n",
            "|    explained_variance   | -2.38e-07   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 39.3        |\n",
            "|    n_updates            | 930         |\n",
            "|    policy_gradient_loss | -0.0036     |\n",
            "|    value_loss           | 38.4        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 574         |\n",
            "|    iterations           | 95          |\n",
            "|    time_elapsed         | 338         |\n",
            "|    total_timesteps      | 194560      |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.010361542 |\n",
            "|    clip_fraction        | 0.0647      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.599      |\n",
            "|    explained_variance   | -3.58e-07   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 0.979       |\n",
            "|    n_updates            | 940         |\n",
            "|    policy_gradient_loss | -0.00166    |\n",
            "|    value_loss           | 38.4        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 574         |\n",
            "|    iterations           | 96          |\n",
            "|    time_elapsed         | 342         |\n",
            "|    total_timesteps      | 196608      |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.011641685 |\n",
            "|    clip_fraction        | 0.106       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.633      |\n",
            "|    explained_variance   | -2.38e-07   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 0.242       |\n",
            "|    n_updates            | 950         |\n",
            "|    policy_gradient_loss | -0.0022     |\n",
            "|    value_loss           | 38.5        |\n",
            "-----------------------------------------\n",
            "------------------------------------------\n",
            "| time/                   |              |\n",
            "|    fps                  | 574          |\n",
            "|    iterations           | 97           |\n",
            "|    time_elapsed         | 345          |\n",
            "|    total_timesteps      | 198656       |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0050521633 |\n",
            "|    clip_fraction        | 0.00498      |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -0.661       |\n",
            "|    explained_variance   | -2.38e-07    |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 2.92         |\n",
            "|    n_updates            | 960          |\n",
            "|    policy_gradient_loss | -0.00046     |\n",
            "|    value_loss           | 38.5         |\n",
            "------------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 574         |\n",
            "|    iterations           | 98          |\n",
            "|    time_elapsed         | 349         |\n",
            "|    total_timesteps      | 200704      |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.017287951 |\n",
            "|    clip_fraction        | 0.0841      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -0.632      |\n",
            "|    explained_variance   | -1.19e-07   |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 5.45        |\n",
            "|    n_updates            | 970         |\n",
            "|    policy_gradient_loss | -0.00294    |\n",
            "|    value_loss           | 38.4        |\n",
            "-----------------------------------------\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<stable_baselines3.ppo.ppo.PPO at 0x7f7c6265e350>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5dEKrz9UYHOz",
        "outputId": "40858ecb-00b8-41b2-dcfd-8b213d05fb22"
      },
      "source": [
        "obs = env.reset()\n",
        "l = []\n",
        "while True:\n",
        "    action, _states = model.predict(obs)\n",
        "    obs, rewards, done, info = env.step(action)\n",
        "    l.append(env.envs[0].select_hash)\n",
        "    if done: \n",
        "        print('info', info)\n",
        "        break\n",
        "selected_hash_PPO = l[-2]\n",
        "Optimized_fees_PPO , Optimized_weight  = info[0]['terminal_observation']\n",
        "env.close()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "info [{'terminal_observation': array([3444175, 4000584])}]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HwPI7fHWuVkX"
      },
      "source": [
        "textfile = open(\"Fees_optimied_PPO.txt\", \"a+\")\n",
        "\n",
        "for element in selected_hash_PPO:\n",
        "\n",
        "    textfile.write(element + \"\\n\")\n",
        "\n",
        "textfile.close()"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FbVvcd6zcbo2",
        "outputId": "f0f28cd9-ef23-40d2-bf31-08d34dbc3500"
      },
      "source": [
        "model = DQN('MlpPolicy', env, verbose = 1)\n",
        "model.learn(total_timesteps=200000)\n"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using cpu device\n",
            "----------------------------------------\n",
            "| rollout/                |            |\n",
            "|    exploration rate     | 0.607      |\n",
            "| time/                   |            |\n",
            "|    episodes             | 4          |\n",
            "|    fps                  | 1407       |\n",
            "|    time_elapsed         | 5          |\n",
            "|    total timesteps      | 8281       |\n",
            "| train/                  |            |\n",
            "|    approx_kl            | 0.00918376 |\n",
            "|    clip_fraction        | 0.0775     |\n",
            "|    clip_range           | 0.2        |\n",
            "|    entropy_loss         | -0.654     |\n",
            "|    explained_variance   | -1.19e-07  |\n",
            "|    learning_rate        | 0.0003     |\n",
            "|    loss                 | 24         |\n",
            "|    n_updates            | 980        |\n",
            "|    policy_gradient_loss | -0.00181   |\n",
            "|    value_loss           | 38.4       |\n",
            "----------------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.21     |\n",
            "| time/               |          |\n",
            "|    episodes         | 8        |\n",
            "|    fps              | 1412     |\n",
            "|    time_elapsed     | 11       |\n",
            "|    total timesteps  | 16628    |\n",
            "----------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.05     |\n",
            "| time/               |          |\n",
            "|    episodes         | 12       |\n",
            "|    fps              | 1415     |\n",
            "|    time_elapsed     | 17       |\n",
            "|    total timesteps  | 24956    |\n",
            "----------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.05     |\n",
            "| time/               |          |\n",
            "|    episodes         | 16       |\n",
            "|    fps              | 1417     |\n",
            "|    time_elapsed     | 23       |\n",
            "|    total timesteps  | 33120    |\n",
            "----------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.05     |\n",
            "| time/               |          |\n",
            "|    episodes         | 20       |\n",
            "|    fps              | 1419     |\n",
            "|    time_elapsed     | 28       |\n",
            "|    total timesteps  | 40806    |\n",
            "----------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.05     |\n",
            "| time/               |          |\n",
            "|    episodes         | 24       |\n",
            "|    fps              | 1417     |\n",
            "|    time_elapsed     | 34       |\n",
            "|    total timesteps  | 48975    |\n",
            "----------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.05     |\n",
            "| time/               |          |\n",
            "|    episodes         | 28       |\n",
            "|    fps              | 1259     |\n",
            "|    time_elapsed     | 44       |\n",
            "|    total timesteps  | 56237    |\n",
            "| train/              |          |\n",
            "|    learning_rate    | 0.0001   |\n",
            "|    loss             | 493      |\n",
            "|    n_updates        | 1559     |\n",
            "----------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.05     |\n",
            "| time/               |          |\n",
            "|    episodes         | 32       |\n",
            "|    fps              | 1131     |\n",
            "|    time_elapsed     | 56       |\n",
            "|    total timesteps  | 64267    |\n",
            "| train/              |          |\n",
            "|    learning_rate    | 0.0001   |\n",
            "|    loss             | 1.09e+03 |\n",
            "|    n_updates        | 3566     |\n",
            "----------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.05     |\n",
            "| time/               |          |\n",
            "|    episodes         | 36       |\n",
            "|    fps              | 1051     |\n",
            "|    time_elapsed     | 68       |\n",
            "|    total timesteps  | 72128    |\n",
            "| train/              |          |\n",
            "|    learning_rate    | 0.0001   |\n",
            "|    loss             | 1.32e+03 |\n",
            "|    n_updates        | 5531     |\n",
            "----------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.05     |\n",
            "| time/               |          |\n",
            "|    episodes         | 40       |\n",
            "|    fps              | 992      |\n",
            "|    time_elapsed     | 80       |\n",
            "|    total timesteps  | 80057    |\n",
            "| train/              |          |\n",
            "|    learning_rate    | 0.0001   |\n",
            "|    loss             | 556      |\n",
            "|    n_updates        | 7514     |\n",
            "----------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.05     |\n",
            "| time/               |          |\n",
            "|    episodes         | 44       |\n",
            "|    fps              | 950      |\n",
            "|    time_elapsed     | 92       |\n",
            "|    total timesteps  | 87943    |\n",
            "| train/              |          |\n",
            "|    learning_rate    | 0.0001   |\n",
            "|    loss             | 538      |\n",
            "|    n_updates        | 9485     |\n",
            "----------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.05     |\n",
            "| time/               |          |\n",
            "|    episodes         | 48       |\n",
            "|    fps              | 917      |\n",
            "|    time_elapsed     | 104      |\n",
            "|    total timesteps  | 96119    |\n",
            "| train/              |          |\n",
            "|    learning_rate    | 0.0001   |\n",
            "|    loss             | 564      |\n",
            "|    n_updates        | 11529    |\n",
            "----------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.05     |\n",
            "| time/               |          |\n",
            "|    episodes         | 52       |\n",
            "|    fps              | 890      |\n",
            "|    time_elapsed     | 117      |\n",
            "|    total timesteps  | 104339   |\n",
            "| train/              |          |\n",
            "|    learning_rate    | 0.0001   |\n",
            "|    loss             | 741      |\n",
            "|    n_updates        | 13584    |\n",
            "----------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.05     |\n",
            "| time/               |          |\n",
            "|    episodes         | 56       |\n",
            "|    fps              | 869      |\n",
            "|    time_elapsed     | 129      |\n",
            "|    total timesteps  | 112400   |\n",
            "| train/              |          |\n",
            "|    learning_rate    | 0.0001   |\n",
            "|    loss             | 731      |\n",
            "|    n_updates        | 15599    |\n",
            "----------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.05     |\n",
            "| time/               |          |\n",
            "|    episodes         | 60       |\n",
            "|    fps              | 852      |\n",
            "|    time_elapsed     | 141      |\n",
            "|    total timesteps  | 120413   |\n",
            "| train/              |          |\n",
            "|    learning_rate    | 0.0001   |\n",
            "|    loss             | 771      |\n",
            "|    n_updates        | 17603    |\n",
            "----------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.05     |\n",
            "| time/               |          |\n",
            "|    episodes         | 64       |\n",
            "|    fps              | 839      |\n",
            "|    time_elapsed     | 153      |\n",
            "|    total timesteps  | 128439   |\n",
            "| train/              |          |\n",
            "|    learning_rate    | 0.0001   |\n",
            "|    loss             | 369      |\n",
            "|    n_updates        | 19609    |\n",
            "----------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.05     |\n",
            "| time/               |          |\n",
            "|    episodes         | 68       |\n",
            "|    fps              | 826      |\n",
            "|    time_elapsed     | 165      |\n",
            "|    total timesteps  | 136551   |\n",
            "| train/              |          |\n",
            "|    learning_rate    | 0.0001   |\n",
            "|    loss             | 1.31e+03 |\n",
            "|    n_updates        | 21637    |\n",
            "----------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.05     |\n",
            "| time/               |          |\n",
            "|    episodes         | 72       |\n",
            "|    fps              | 815      |\n",
            "|    time_elapsed     | 178      |\n",
            "|    total timesteps  | 145175   |\n",
            "| train/              |          |\n",
            "|    learning_rate    | 0.0001   |\n",
            "|    loss             | 415      |\n",
            "|    n_updates        | 23793    |\n",
            "----------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.05     |\n",
            "| time/               |          |\n",
            "|    episodes         | 76       |\n",
            "|    fps              | 806      |\n",
            "|    time_elapsed     | 189      |\n",
            "|    total timesteps  | 153139   |\n",
            "| train/              |          |\n",
            "|    learning_rate    | 0.0001   |\n",
            "|    loss             | 605      |\n",
            "|    n_updates        | 25784    |\n",
            "----------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.05     |\n",
            "| time/               |          |\n",
            "|    episodes         | 80       |\n",
            "|    fps              | 797      |\n",
            "|    time_elapsed     | 201      |\n",
            "|    total timesteps  | 161038   |\n",
            "| train/              |          |\n",
            "|    learning_rate    | 0.0001   |\n",
            "|    loss             | 1.1e+03  |\n",
            "|    n_updates        | 27759    |\n",
            "----------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.05     |\n",
            "| time/               |          |\n",
            "|    episodes         | 84       |\n",
            "|    fps              | 790      |\n",
            "|    time_elapsed     | 213      |\n",
            "|    total timesteps  | 168982   |\n",
            "| train/              |          |\n",
            "|    learning_rate    | 0.0001   |\n",
            "|    loss             | 693      |\n",
            "|    n_updates        | 29745    |\n",
            "----------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.05     |\n",
            "| time/               |          |\n",
            "|    episodes         | 88       |\n",
            "|    fps              | 784      |\n",
            "|    time_elapsed     | 224      |\n",
            "|    total timesteps  | 176342   |\n",
            "| train/              |          |\n",
            "|    learning_rate    | 0.0001   |\n",
            "|    loss             | 782      |\n",
            "|    n_updates        | 31585    |\n",
            "----------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.05     |\n",
            "| time/               |          |\n",
            "|    episodes         | 92       |\n",
            "|    fps              | 779      |\n",
            "|    time_elapsed     | 236      |\n",
            "|    total timesteps  | 184284   |\n",
            "| train/              |          |\n",
            "|    learning_rate    | 0.0001   |\n",
            "|    loss             | 489      |\n",
            "|    n_updates        | 33570    |\n",
            "----------------------------------\n",
            "----------------------------------\n",
            "| rollout/            |          |\n",
            "|    exploration rate | 0.05     |\n",
            "| time/               |          |\n",
            "|    episodes         | 96       |\n",
            "|    fps              | 773      |\n",
            "|    time_elapsed     | 248      |\n",
            "|    total timesteps  | 192080   |\n",
            "| train/              |          |\n",
            "|    learning_rate    | 0.0001   |\n",
            "|    loss             | 912      |\n",
            "|    n_updates        | 35519    |\n",
            "----------------------------------\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<stable_baselines3.dqn.dqn.DQN at 0x7f7c07fad890>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lE3rjtWmqvAn",
        "outputId": "8c5f73ba-bf3f-463f-9950-bfc8df78d99d"
      },
      "source": [
        "obs = env.reset()\n",
        "c = []\n",
        "while True:\n",
        "    action, _states = model.predict(obs)\n",
        "    obs, rewards, done, info = env.step(action)\n",
        "    c.append(env.envs[0].select_hash)\n",
        "    if done: \n",
        "        print('info', info)\n",
        "        break\n",
        "selected_hash_DQN = c[-2]\n",
        "Optimized_fees_DQN , Optimized_weight_DQN  = info[0]['terminal_observation']\n",
        "env.close()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "info [{'terminal_observation': array([3096998, 4011052])}]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jpe_76TJhqaN"
      },
      "source": [
        "textfile = open(\"Fees_optimied_DQN.txt\", \"a+\")\n",
        "\n",
        "for element in selected_hash_DQN:\n",
        "\n",
        "    textfile.write(element + \"\\n\")\n",
        "\n",
        "textfile.close()"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_dmNOEYFviOt"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}