3.grid_world_reinforcement.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importação dos módulos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import random, operator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Funções utilitárias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def argmax(seq, fn):\n",
    "    best = seq[0]\n",
    "    best_score = fn(best)\n",
    "    for x in seq:\n",
    "        x_score = fn(x)\n",
    "        if x_score > best_score:\n",
    "            best, best_score = x, x_score\n",
    "    return best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def vector_add(a, b):\n",
    "    return tuple(map(operator.add, a, b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def isnumber(x):\n",
    "    return hasattr(x, '__int__')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ações do agente"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Conjunto de orientações possíveis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Funções de rotação do agente"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Definição do modelo (Markov decision process)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class MDP:\n",
    "    def __init__(self, init_pos, actlist, terminals, transitions={}, states=None, gamma=0.99):\n",
    "        if not (0 < gamma <= 1):\n",
    "            raise ValueError('Gamma value')\n",
    "        if states:\n",
    "            self.states = states\n",
    "        else:\n",
    "            self.states = set()\n",
    "            self.init_pos = init_pos\n",
    "            self.actlist = actlist\n",
    "            self.terminals = terminals\n",
    "            self.transitions = transitions\n",
    "            self.gamma = gamma\n",
    "            self.reward = {}\n",
    "            \n",
    "    def R(self, state):\n",
    "        return self.reward[state]\n",
    "    \n",
    "    def T(self, state, action):\n",
    "        if (self.transitions == {}):\n",
    "            raise ValueError('Transition model missing')\n",
    "        else:\n",
    "            return self.transitions[state][action]\n",
    "        \n",
    "    def actions(self, state):\n",
    "        if state in self.terminals:\n",
    "            return [None]\n",
    "        else:\n",
    "            return self.actlist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Definição do ambiente (grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class GridMDP(MDP):\n",
    "    def __init__(self, grid, terminals, init_pos=(0, 0), gamma=0.99):\n",
    "        grid.reverse()\n",
    "        MDP.__init__(self, init_pos, actlist=orientations, terminals=terminals, gamma=gamma)\n",
    "        self.grid = grid\n",
    "        self.rows = len(grid)\n",
    "        self.cols = len(grid[0])\n",
    "        for x in range(self.cols):\n",
    "            for y in range(self.rows):\n",
    "                self.reward[x, y] = grid[y][x]\n",
    "                if grid[y][x] is not None:\n",
    "                    self.states.add((x, y))\n",
    "                    \n",
    "    def T(self, state, action):\n",
    "        # Função de transição de estado dada a ação do agente\n",
    "        \n",
    "    def go(self, state, direction):\n",
    "        # Função de movimentação do agente\n",
    "    \n",
    "    def to_grid(self, mapping):\n",
    "        return list(reversed([[mapping.get((x, y), None)\n",
    "                              for x in range(self.cols)]\n",
    "                             for y in range(self.rows)]))\n",
    "    \n",
    "    def to_arrows(self, policy):\n",
    "        chars = {\n",
    "            (1, 0): '>',\n",
    "            (0, 1): '^',\n",
    "            (-1, 0): '<',\n",
    "            (0, -1): 'v',\n",
    "            None: '.'\n",
    "        }\n",
    "        return self.to_grid({\n",
    "            s: chars[a] for (s, a) in policy.items()\n",
    "        })"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Value iteration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def value_iteration(mdp, epsilon=0.001):\n",
    "    # Algoritmo de value iteration\n",
    "\n",
    "def best_policy(mdp, STS):\n",
    "    # Seleção da melhor política"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Policy iteration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def expected_utility(a, s, STS, mdp):\n",
    "    # Função do valor esperado\n",
    "\n",
    "def policy_iteration(mdp):\n",
    "    # Algoritmo de policy iteration\n",
    "\n",
    "def policy_evaluation(pi, STS, mdp, k=20):\n",
    "    # Função de avaliação da política"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Impressão do grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def print_table(table, header=None, sep=' ', numfmt='{}'):\n",
    "    justs = ['rjust' if isnumber(x) else 'ljust' for x in table[0]]\n",
    "    if header:\n",
    "        table.insert(0, header)\n",
    "    table = [[numfmt.format(x) if isnumber(x) else x for x in row]\n",
    "            for row in table]\n",
    "    sizes = list(map(lambda seq: max(map(len, seq)),\n",
    "                    list(zip(*[map(str, row) for row in table]))))\n",
    "    for row in table:\n",
    "        print(sep.join(getattr(str(x), j)(size) for (j, size, x)\n",
    "                      in zip(justs, sizes, row)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Criação do ambiente"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Criação do ambiente"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Solução por value iteration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Solução do grid por value iteration\n",
    "\n",
    "print('Optimal Policy based on value iteration')\n",
    "print_table(sequential_decision_environment.to_arrows(value_iter))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Solução por policy iteration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Solução do grid por policy iteration\n",
    "\n",
    "print('Optimal Policy based on policy iteration and evaluation')\n",
    "print_table(sequential_decision_environment.to_arrows(policy_iter))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}