Snake_PPO.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c86b7e46-ebd2-4f4c-92d2-e93bc5bf9c0c",
   "metadata": {},
   "source": [
    "ALL IN ONE IPYNB FILE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a9b6009a-c4fa-4a52-bedf-2ddf34328df3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import gym\n",
    "from gym import spaces\n",
    "\n",
    "import pygame\n",
    "import random\n",
    "import numpy as np\n",
    "\n",
    "import traceback\n",
    "\n",
    "WIDTH, HEIGHT = 1005,  510\n",
    "get_random_apple = lambda: [random.randrange(1,int(WIDTH/15))*15,random.randrange(1,int(HEIGHT/15))*15]\n",
    "\n",
    "class Snake:\n",
    "\n",
    "    def __init__(self):\n",
    "        \n",
    "        x = random.randint(30, WIDTH-50)\n",
    "        x -= x % 15 or 0\n",
    "        #x = (lambda num: num - num % 15 or 1)(random.randint(100, 1000))\n",
    "        \n",
    "        y = random.randint(100, HEIGHT-50)\n",
    "        y -= y % 15 or 0\n",
    "        \n",
    "        self.head = [x,y]\n",
    "        self.tail = [[x+15,y],[x+30,y],[x+45,y]]\n",
    "        \n",
    "        \n",
    "class SnakeEnv(gym.Env):\n",
    "    metadata = {'render.modes': ['human']}\n",
    "\n",
    "    def __init__(self):\n",
    "        super(SnakeEnv, self).__init__()\n",
    "        \n",
    "        self.done = False #For episode \n",
    "        self.run = True # For game window\n",
    "        \n",
    "        self.action_space = spaces.Discrete(4) #For left right up and down\n",
    "        self.observation_space = spaces.Box(low=0, high=1, shape=(5,), dtype=float) # 5 = Snake_X, Snake_Y, Apple_X, Apple_Y, Euclid_Distance_Apple_to_Snake\n",
    "        self.info = {}\n",
    "        \n",
    "        self.Agent = Snake()\n",
    "        self.Apple = get_random_apple()\n",
    "        \n",
    "        self.distance = np.linalg.norm(np.array([self.Agent.head[0], self.Agent.head[1]]) - np.array([self.Apple[0], self.Apple[1]]))\n",
    "    \n",
    "    \n",
    "    def step(self, action):\n",
    "    \n",
    "        self.render()\n",
    "        \n",
    "        reward = 0\n",
    "        \n",
    "        if action == 0: #left\n",
    "            self.Agent.head[0] -= 15\n",
    "    \n",
    "        if action == 1: #up\n",
    "            self.Agent.head[1] -= 15\n",
    "    \n",
    "        if action == 2: #right\n",
    "            self.Agent.head[0] += 15\n",
    "    \n",
    "        if action == 3: # down\n",
    "            self.Agent.head[1] += 15\n",
    "    \n",
    "        if np.linalg.norm(np.array([self.Agent.head[0], self.Agent.head[1]]) - np.array([self.Apple[0], self.Apple[1]])) < self.distance:\n",
    "            reward += 15\n",
    "        \n",
    "        else:\n",
    "            reward -= 5\n",
    "        \n",
    "        self.distance = np.linalg.norm(np.array([self.Agent.head[0], self.Agent.head[1]]) - np.array([self.Apple[0], self.Apple[1]]))\n",
    "        \n",
    "        self.observation = np.array([self.Agent.head[0], self.Agent.head[1], self.Apple[0], self.Apple[1], self.distance])\n",
    "        \n",
    "        if self.Agent.head[1] < 0 or self.Agent.head[0] < 0 or self.Agent.head[1] > HEIGHT or self.Agent.head[0] > WIDTH:\n",
    "            self.done = True\n",
    "       \n",
    "        if self.Apple == self.Agent.head:\n",
    "            \n",
    "            self.Agent.tail.insert(0,list(self.Agent.head))\n",
    "            self.Apple = get_random_apple()\n",
    "            self.distance = np.linalg.norm(np.array([self.Agent.head[0], self.Agent.head[1]]) - np.array([self.Apple[0], self.Apple[1]]))\n",
    "                \n",
    "        \n",
    "        return self.observation, reward, self.done, self.info\n",
    "        \n",
    "    \n",
    "    def reset(self):\n",
    "    \n",
    "        self.Agent = Snake()\n",
    "        self.Apple = get_random_apple()\n",
    "        \n",
    "        self.distance = np.linalg.norm(np.array([self.Agent.head[0], self.Agent.head[1]]) - np.array([self.Apple[0], self.Apple[1]]))\n",
    "    \n",
    "        self.observation = np.array([self.Agent.head[0], self.Agent.head[1], self.Apple[0], self.Apple[1], self.distance])\n",
    "        self.done = False\n",
    "        \n",
    "        return self.observation\n",
    "    \n",
    "    \n",
    "    def render(self):\n",
    "        \n",
    "        screen.fill((0,0,0))\n",
    "        pygame.draw.rect(screen, (255,0,0), [self.Apple[0],self.Apple[1], 15,15])\n",
    "\n",
    "        for pos in self.Agent.tail:\n",
    "            pygame.draw.rect(screen, (0,0,120), [pos[0], pos[1], 15,15])\n",
    "\n",
    "        self.Agent.tail.insert(0,list(self.Agent.head))\n",
    "        self.Agent.tail.pop()\n",
    "\n",
    "        \n",
    "        blockSize = 15 \n",
    "        for x in range(0, 1200, blockSize):\n",
    "            for y in range(75, 800, blockSize):\n",
    "                rect = pygame.Rect(x, y, blockSize, blockSize)\n",
    "                pygame.draw.rect(screen, (25,25,25), rect, 1)\n",
    "        \n",
    "        pygame.display.update()\n",
    "        \n",
    "        for event in pygame.event.get():\n",
    "            if event.type == pygame.QUIT:\n",
    "                self.run = False\n",
    "                self.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6a74582e-3b08-4ea5-8e40-ab7860144e4d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 1 Score:2790\n"
     ]
    }
   ],
   "source": [
    "env = SnakeEnv()\n",
    "screen = pygame.display.set_mode((WIDTH, HEIGHT))\n",
    "\n",
    "episode = 10\n",
    "\n",
    "while episode > 0 and env.run:\n",
    "    state = env.reset()\n",
    "    env.done = False\n",
    "    score = 0\n",
    "    \n",
    "    while not env.done and env.run:\n",
    "     \n",
    "        env.render()\n",
    "        action = env.action_space.sample()\n",
    "        n_state, reward, done, info = env.step(action)\n",
    "        score += reward\n",
    "        \n",
    "    print('Episode: {} Score:{}'.format(11 - episode , score))\n",
    "\n",
    "pygame.display.quit()\n",
    "pygame.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "babef4a3-d859-4f66-b831-d7081bf50061",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\gokss\\anaconda3\\envs\\mech\\lib\\site-packages\\stable_baselines3\\common\\vec_env\\patch_gym.py:49: UserWarning: You provided an OpenAI Gym environment. We strongly recommend transitioning to Gymnasium environments. Stable-Baselines3 is automatically wrapping your environments in a compatibility layer, which could potentially cause issues.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from stable_baselines3 import PPO, A2C\n",
    "import traceback\n",
    "import sys\n",
    "\n",
    "\n",
    "WIDTH, HEIGHT = 1005,  510\n",
    "screen = pygame.display.set_mode((WIDTH,HEIGHT))\n",
    "\n",
    "\n",
    "env = SnakeEnv()\n",
    "\n",
    "try:\n",
    "    model = A2C('MlpPolicy',env, learning_rate=0.0004, gamma=0.99, gae_lambda=0.95, max_grad_norm=0.5, verbose=0)\n",
    "    \n",
    "    #policy, env, learning_rate=0.0003, n_steps=2048, batch_size=64, n_epochs=10, gamma=0.99, gae_lambda=0.95, clip_range=0.2, \n",
    "    #clip_range_vf=None, normalize_advantage=True, ent_coef=0.0, vf_coef=0.5, max_grad_norm=0.5, use_sde=False, sde_sample_freq=-1, \n",
    "    #target_kl=None, stats_window_size=100, tensorboard_log=None, policy_kwargs=None, verbose=0, seed=None, device='auto', _init_setup_model=True\n",
    "    \n",
    "    model.learn(total_timesteps=500000, callback=None, log_interval=1, tb_log_name='PPO', reset_num_timesteps=True, progress_bar=False)\n",
    "    \n",
    "    \"\"\"\n",
    "    gamma = A higher gamma value (e.g., 0.99) means that the agent considers future rewards more heavily, which can lead to a longer-term focus in its learning. Conversely, a lower gamma value (e.g., 0.9) makes the agent prioritize immediate rewards.\n",
    "    gae_lambda = is the factor used for trade-off between bias and variance when estimating advantages with the Generalized Advantage Estimator (GAE). GAE is used to estimate how much better or worse taking a specific action is compared to following the current policy. A higher gae_lambda value (e.g., 0.95) reduces bias but may increase variance, while a lower value reduces variance but may introduce bias into the advantage estimates.\n",
    "    max_grad_norm = is the maximum allowed value for gradient clipping during the training process. Gradient clipping is a technique used to prevent exploding gradients, which can lead to unstable training. If the magnitude of the gradients exceeds this value, they are scaled down to ensure they do not become too large.\n",
    "    \"\"\"\n",
    "    \n",
    "except Exception as e:\n",
    "    exc_type, exc_obj, tb = sys.exc_info()\n",
    "    line_number = tb.tb_lineno\n",
    "    print(\"!Error at\", line_number,\"th Line\")\n",
    "    traceback.print_exc()\n",
    "    pygame.quit()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1fa14ba-a3ae-46bd-858c-a82b1bd1a7c5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0193685-db92-4da6-a28f-8def5a3b8e41",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mech",
   "language": "python",
   "name": "mech"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}