tpg_examples.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TPG Examples\n",
    "This document shows how to use the PyTPG API. We make use of OpenAI Gym to run examples, and we assume you already have PyTPG installed, see the readme for installation instructions for PyTPG."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## General Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "from IPython.display import clear_output\n",
    "import numpy as np\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "from IPython import display\n",
    "import random\n",
    "\n",
    "# how to render in Jupyter: \n",
    "# https://stackoverflow.com/questions/40195740/how-to-run-openai-gym-render-over-a-server\n",
    "# https://www.youtube.com/watch?v=O84KgRt6AJI\n",
    "def show_state(env, step=0, name='', info=''):\n",
    "    plt.figure(3)\n",
    "    plt.clf()\n",
    "    plt.imshow(env.render(mode='rgb_array'))\n",
    "    plt.title(\"%s | Step: %d %s\" % (name, step, info))\n",
    "    plt.axis('off')\n",
    "\n",
    "    display.clear_output(wait=True)\n",
    "    display.display(plt.gcf())\n",
    "    \n",
    "# To transform pixel matrix to a single vector.\n",
    "def getState(inState):\n",
    "    # each row is all 1 color\n",
    "    rgbRows = np.reshape(inState,(len(inState[0])*len(inState), 3)).T\n",
    "\n",
    "    # add each with appropriate shifting\n",
    "    # get RRRRRRRR GGGGGGGG BBBBBBBB\n",
    "    return np.add(np.left_shift(rgbRows[0], 16),\n",
    "        np.add(np.left_shift(rgbRows[1], 8), rgbRows[2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "env = gym.make('Assault-v0') # make the environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Discrete(7)\n"
     ]
    }
   ],
   "source": [
    "print(env.action_space) # learn size of action space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import to do training\n",
    "from tpg.trainer import Trainer\n",
    "# import to run an agent (always needed)\n",
    "from tpg.agent import Agent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Example: Basic Generational Selection (with graphics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATQAAAD3CAYAAACTiqgxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAASz0lEQVR4nO3da7RcZX3H8e8fopCiEAEDhIsgXiMoK4giCo2AerQqojXUG0QXSl5QqgtdovWCliVoVXR5o4g2tgElohXrJVVAXNEQZRkDlYAIQlA0FORiiMFaePrieSZshpk5c86ZOZcn389aZ509+/rsPXt++3mePefsSCkhSTXYZqoLIEmDYqBJqoaBJqkaBpqkahhokqphoEmqxowLtIi4PCJOHMP8iyNi6RCLJGma6DvQSpDcFRHbDbNAY1HC6kcTXMfzImJVRNwTEXdGxI8j4pBBrX+cZdo5Im5v33ZEnBgRN0TEvRGxIiLmNabNiYgvRcT/lJ/TR9nGIyPifRHxy4jYFBG3RsR3I+KFQ9qt5rZvjYjZEXFkRHy9yzx/HREpIs7oc533Nn4eiIjNjdevG+wejE9kt0XEIyLiJRFxQdv0D0fELyLi/og4bYzrfkZEXFo+o3dFxJUR8YLB7sHERcSLI+L6cs5dEhF79Zh3/4hYGRF/iohrIuKI0dbfV6BFxL7A4UACXt5n2ae9iNgR+BbwKWBnYE/gA8Cfp7JcwIeBa5sjImIh8CHgGHJZbwK+3JjlbOCvgH2BZwFviIg39tjGRWVdxwOPAfYDPgn8zSB2oJuI2Bv4Q0ppM3AwsKbDPI8oZflJv+tNKT2q9QPcArysMe78ARV/op4ArE8p/YXO+/5L4FTg+2NZaURsA3wbuBiYC+xe1nPvRAvctp1ZE1x+D+BC4B3ArsA6YFmPRS4CVpLP9zOAb0TEnJ4bSSmN+gO8D/gx8HHgW23TXlIKthG4FXh7Gb8rOSzuBu4sBdumTDsNuLEssw44trG+04Fljdf7koN0Vnl9OXAi8FTgPuB+8ht3d5eyLwaWdpn2zB7LdVw/sB3wUfKH5jbgHGB2mbYQ+C3wbuAO4Gbgdf0c48Z2DwOuAN4I/Kgx/qPAZxqv55Xjsn95fQdwSGP6u4GVXbZxNLAZ2GuUsswDvgbcTg7QU9rep+XAv5X38RrgmX3s37HAv5bhC4GXdJjnNOAjwFLgjLEcv7L8zcDRbeO2Bd4L/Locq/OBOWXarLKft5Xz9QfAkxvLfoUcsN8HNpVzcC7w2TL/NcCBfZTrNcDnyvDFwJFd5rsIOG0M+7tXORdm95jn1cDV5b36FXBUGb8P8B3yZ/R64ITGMmcBF5T3aSPw+l7HsY9yngJc1ng9B/hfYN8O8z69HOvZjXFXAot7baPfJufxpeDnAy+KiN0a074AnJRSejRwAHBZGX8q+cP9WGA38ges9XdWN5JrfDuRa0TLSnr3LaV0LbAEuCLlq3Dv5O7seuD+0lR7cUQ8po/1nwU8CTiIfMXdkxz4LbuTw3xP4ATg3Ih4MkBEvDYiru5WmIjYFvg0cDIPHquHzNJh+IAe05vTmo4GfpJS+m2PsmwD/CdwVdmXo4C3RsSLGrO9nPxhnwN8s5S92/reHxF3k2uVx5XhvwW+HBF3l30nIh4HvAn4YLd1jdPbgRcCzyMHwF/ItdqWi4H9ye/fdcCX2pY/rqxjV3IArgZ+COxCDoSPdNtwRJxZ9ncpcEIZfinwHxFxx0R3DNgArAcuiIhjImJu2/aPAM4F/oH8mTsK+E2Z/FVyzXAP4LXA2RHx3MbiryIfi53Iod/zOJYujFd2KefTyOcTACmlu8kVg6d1mff6lGvyLVd1mfdBfaTq80qhdy2vrwPe1ph+C3ASsGPbch8knyRP6GMba4FjGlf+UWtoZXgxjVpMl3UvpksNrUx/KvlE+y3wf+QP5m6d1k8OiU2UWlEZ9xzgpjK8sKxjh8b05cB7+7yCvY0Hr+Dt2z6afEV8OjAb+BfgAeA1Zfoy4OvAo8lBeyPw5y7bOQ/4SuP1zuSaxj3AfWXcs4Fb2pZ7Fw/Wrk4HLmlMmw9sHmX/ZpGb0ruRa6Lf7jDPxcBxZXgpg6uh3QQ8t/F6P+BPQHRYfvdybLcvr78CfKox/R3AzxuvDwE2jFKmR5KDYxfgSOBrPeYdUw2tLPM4cmvhJnKr4lJgvzLtS8CZHZZ5IrkV0qwFnQ2cU4bPAr433uPYYXvnA6e3jfsZ8Hcd5n0zcHnbuI+1ytbtp58a2gllp1pXkgvKuJZXkZud6yPihxHxnDL+n4EbgO9FxK+bnZwRcXxErC1X5rvJNYld+yjLwKWUrk0pLU4p7VXKMQ/4RJfZH0vup/pZo+wryviWu1JKmxqv15d19lQ6+E8B/rFLOS8B3k++St5cfjaSg5iy7GZyc+Jick2oWw3sD+Qrcmvdd6ZcAz2Y3KSG/AGZ19rPsq/vJodRy4bG8J+A7Tv1s0TEQWX5u8hh+0tys25hWfcry3wvAx6dUrqwS7nHJSIC2Bv4TmNffk7uQ94lImZFxMfKefpH8kU7yOHTcltjeHOH14/qsu1Dy/buJH/4bwS+C4yUsgykzzKltD6ltCSltB/w+DL6i+X33mW77eYBt6eH1oLWk2vkLa2a3KjHsY9i3gvs2DZuR/J5PJF5t+jZyRcRs4FFwLYR0Tp5twPmRMQzUkpXpZSuBI4pHbknk2ske6eUNpKbnadGxAHAZRFxJTnkPk+u9l6RUro/ItbyYHNpEzk0WnbvUcSB/quQlNJ1kb/icVKX9d9BPnmfllK6tctqHhMROzRCbR/gF31s/lnkkFmXzxtmA7PLcd8zpXR/SukzwGcAIuJJwHta604p3QlsuZsXER8CftplW5cCfx8Re6Xuzc7fkGueT+yj7D2llNaSz5nTyFfyMyPi58CrU0o3NGY9Cnhm41zbidwlcGBK6ZgJbD9FxK3AK1NKP2ufHhFvBl4APJ/c4tgN+D0PbcKPd9uryft+OnBPSunsiLgWeEGPYz/Rba6PiM8BnyujfkNuTrf7HfDYiJjdCLV9yH3hW1bXWG/P49iHa8g3ogCIiJ3IF85rusz7pIjYPqV0Xxn3DHK/ZVej1dBeQa6+zif3GR1EbqKtBI6PfOv/dRGxU8p3bv5IrqoTES+NiCeUVL+nrOcBYAfyQbq9zPdGHtrXsxY4IiL2KTv8rh7luw3YKyIeOcp+dBQRT4mIU1u3jssduNeQ+0cetv6U0gPkMD671U8REXu29SsBfKAcm8PJfSVf7aM43yU3r1vH+X3kq99BJfS3j4gDItuH3CfyyZTSXaUc+0fELhGxbUS8GHgL+c7Qw6SUvkeuIX0jIp5dyvoI4NDGbD8FNkbEOyN/xWLbsv1D+tiXbg4G1pTjOa8tzCB3Nrf6Jw8iN/8/T75BQkQsjIjxXsTOAc4q7zERMbfUCCE30+8j11x3oMtxm6DWvu9A7p55WJhF/jrH9uTP5azynm9Tpj0l8tdYHnaBj4jdIn8F5/Hl/JhL7rJoncfnASdFxBERsU1E7F0uiDcA/w2cERHbRcQCcuur153HXsdxNBcBh0TEy8p+fgBYlVK6uX3GlNLV5NbGe0vZjiPX7i/utYHRAu0Ecp/JLSmlDa0fcudvqzbwBuDmUlVf0hj/ROASctXxCuCzKaUfpJTWkdvCV5AD40DyHdTWjnyffFflanL7+ls9yncZOck3xPg6VzeS+4p+EhGbyCfAL8g1y27rfyf5RFhd9vkS4MmNdW4gN61+R+4zWJJSug6ghH+nqxEppT+3HeN7gL+UYYDtyc39e8lhcwU5AFoOJp+cG4EzyXdXO26rOJZ8bJeR+89uIr93LyrluZ8cxgeVaXeQPxg79VjnaFpfVTiQDrXWlNLGtmOwGdhUap+Qmzurxrntj5Dfq8siYmNZz4Iy7QvkC+wG8jEcxncPF1AuUOSLdif/Tt7nY4F/KsOLyrS9yR/w2zssdx/583Y5+f2/inwOngiQUlpJ/mx+lnxeXUq+w53Idz/nk/f9QuAdKaVe+9/rOBIRN0bEqzotmFL6HfnmysfJTfADyHdOW8sujYhmd8+rgSPI5+f7yd+GuKtH2XJHXs0iYjGwMKW0eBK2tZB8Q6PrlwU1fhFxHvDVlNJ/TXVZJlvkLxj/KqXUfvdVDRP6opw0mVJKff/JW21SSu+Z6jLMBFtDoK0lV1klVa76JqekrceM+28bktTN1tDknHQT+GqBtkIppQl/302ZNTRJ1TDQJFXDQJNUDQNNUjUMNEnVMNAkVcNAk1QNA01SNQw0SdUw0CRVw0CTVA0DTVI1DDRJ1TDQJFXDQJNUDQNNUjUMNEnVMNAkVcNAk1QNA01SNQw0SdUw0CRVw0CTVA0DTVI1DDRJ1TDQJFXDQJNUDQNNUjUMNEnVMNAkVcNAk1QNA01SNQw0SdWYNdUF0PS17vWv3zI8f9myaTtNarGGpo5aAdIKj2agTKdpUpOBJqkaBpqkahhokqphoEmqRqSUproM1YmIKg7qdLqTWfNdzpRSTHUZamGgDUEtgabJYaANjk1OSdUw0CRVw0CTVA0DTVI1/FtOVW/1yMiW4UNXrJjCkmjYvMs5BN7lnForP3F43/Me/taVQyxJf7zLOTgG2hAYaJNv9ze9fMLr2PDFbw6gJGNnoA2OgTYEBtrkWTKyauDrPGfFYQNfZy8G2uAYaENgoA3fqpOHHzqHfXrwYdmJgTY43hTQjNIeZK3QWXXyYQ8ZHvQ0zQwGmmakTiHUKZAmMk0zj03OIbDJOTzD6DPrZTL602xyDo6BNgQG2vDt/orTh76NDd8Y/jbAQBskm5yamXZeM9Ul0DRkDW0IrKFNnn6+RNv88uxY558M1tAGx7/llFQNa2hDYA1t8jX/XrOl199tjnX+YbKGNjgG2hAYaFOjFVJjCabVIyNT/gfrBtrgGGhDYKBpLAy0wbEPTVI1DDRJ1TDQJFXDQJNUDQNNUjUMNEnVMNAkVcNAk1QNA01SNQw0SdUw0CRVw0CTVA0DTVI1DDRJ1TDQJFXDQJNUDQNNUjV8jN1WZske524ZPuf3bxnzdGk6s4YmqRoG2lZkyR7nbql1nfP7t2ypjS3Z49yuNbPmeGm68yEpQzCdH5LSCqhWaHUKMpudk8uHpAyOgTYE0znQVixYwMiaNaxYsKDrPK3pzd8aHgNtcGxyysBSNayhDcF0rqG1P4y3+QTxXuM0PNbQBsdAG4LpHGiafgy0wbHJKakaBpqkahhokqphoEmqhoEmqRoGmqRqGGiSqmGgSaqGgSapGgaapGoYaJKqYaBJqoaBJqkaBpqkavjUpxmq17/J9l9oa2tlDU1SNQy0GajX05tafHKTtkb+x9ohmIz/WNvt6U0+uWnm8T/WDo6BNgSTEWijPb3JJzfNHAba4NjkrIihpa2dgTYDrR4ZYc7cuQDMmTuXkTVrtrxuDa8eGWFkzZqHzCvVzibnEPjUJ42FTc7BsYYmqRoGmqRqGGiSqmGgSaqGgSapGgaapGoYaJKqYaBJqoaBJqkaBpqkahhokqphoEmqhoEmqRoGmqRqGGgauCV7nNv1OQa9pkkTZaBJqoaBpoFqPpGq+TSqTg9tsaamQfNBwxqYVpg1n0DVHlrt06VB8l9wD8HW+i+4258y1XoiVfvTqVoPc/FpVJn/gntwrKFpYJoPblk9MsKhK1awemSE1SMjABy6YgXAw15Lg2IfmgaqU1g1h1vT24elQbDJOQRba5NT42OTc3CsoUmqhoEmqRoGmqRqGGiSqmGgSaqGgSapGgaapGoYaJKqYaBJqoaBJqkaBpqkahhokqphoEmqhoEmqRoGmqRqGGiSqmGgSaqGgSapGgaapGoYaJKqYaBJqoaBJqkaBpqkahhokqphoEmqhoEmqRoGmqRqGGiSqmGgSaqGgSapGgaapGoYaJKqYaBJqoaBJqkaBpqkahhokqphoEmqhoEmqRoGmqRqGGiSqmGgSaqGgSapGgaapGoYaJKqYaBJqoaBJqkaBpqkahhokqphoEmqhoEmqRoGmqRqGGiSqmGgSaqGgSapGgaapGoYaJKqYaBJqoaBJvUwsnSEkaUjU10M9SlSSlNdhupEhAe1MiNLR1ixeMVQ1p1SiqGseCtkoA2BgVaHdevWMX/+/C3DwJbXg2SgDY6BNgQG2szXCrBOBh1qBtrgzJrqAkjT0a+XvXaqi6Bx8KaA1MFLP7R2y+/WcHO8picDTeqiPbwMs+nPPrQhsA9NY2Ef2uDYhzYEp5x35FQXQdoq2eSUVA2bnENgk1NjYZNzcKyhSaqGgSapGt4UmIaWL1/OokWLpv1yk2XVyYd1nXbYp1dNYkk03Rlo08jy5csBWLRo0UOGp9ty0nTlTYEhGM9NgVagdNIrZCZ7ualQew3NmwKDYx+apGoYaNNEs1bUbXg6LCdNZwbaNNIeJv2Gy2QvJ01X9qENwXi/WNupX2ssnfuTtdxksw9N/TLQhsC/FNBYGGiDY5NTUjUMNEnVMNAkVcNAk1QNA01SNQw0SdUw0CRVw0CTVA0DTVI1DDRJ1fBPnyRVwxqapGoYaJKqYaBJqoaBJqkaBpqkahhokqrx/20k7fLSWqbsAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time Taken (Hours): 0.6057909552256266\n",
      "Results:\n",
      "Min, Max, Avg\n",
      "0.0 210.0 10.5\n",
      "0.0 210.0 10.5\n",
      "0.0 357.0 27.0\n",
      "0.0 399.0 51.45\n",
      "0.0 378.0 74.55\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATQAAAD3CAYAAACTiqgxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAASz0lEQVR4nO3da7RcZX3H8e8fopCiEAEDhIsgXiMoK4giCo2AerQqojXUG0QXSl5QqgtdovWCliVoVXR5o4g2tgElohXrJVVAXNEQZRkDlYAIQlA0FORiiMFaePrieSZshpk5c86ZOZcn389aZ509+/rsPXt++3mePefsSCkhSTXYZqoLIEmDYqBJqoaBJqkaBpqkahhokqphoEmqxowLtIi4PCJOHMP8iyNi6RCLJGma6DvQSpDcFRHbDbNAY1HC6kcTXMfzImJVRNwTEXdGxI8j4pBBrX+cZdo5Im5v33ZEnBgRN0TEvRGxIiLmNabNiYgvRcT/lJ/TR9nGIyPifRHxy4jYFBG3RsR3I+KFQ9qt5rZvjYjZEXFkRHy9yzx/HREpIs7oc533Nn4eiIjNjdevG+wejE9kt0XEIyLiJRFxQdv0D0fELyLi/og4bYzrfkZEXFo+o3dFxJUR8YLB7sHERcSLI+L6cs5dEhF79Zh3/4hYGRF/iohrIuKI0dbfV6BFxL7A4UACXt5n2ae9iNgR+BbwKWBnYE/gA8Cfp7JcwIeBa5sjImIh8CHgGHJZbwK+3JjlbOCvgH2BZwFviIg39tjGRWVdxwOPAfYDPgn8zSB2oJuI2Bv4Q0ppM3AwsKbDPI8oZflJv+tNKT2q9QPcArysMe78ARV/op4ArE8p/YXO+/5L4FTg+2NZaURsA3wbuBiYC+xe1nPvRAvctp1ZE1x+D+BC4B3ArsA6YFmPRS4CVpLP9zOAb0TEnJ4bSSmN+gO8D/gx8HHgW23TXlIKthG4FXh7Gb8rOSzuBu4sBdumTDsNuLEssw44trG+04Fljdf7koN0Vnl9OXAi8FTgPuB+8ht3d5eyLwaWdpn2zB7LdVw/sB3wUfKH5jbgHGB2mbYQ+C3wbuAO4Gbgdf0c48Z2DwOuAN4I/Kgx/qPAZxqv55Xjsn95fQdwSGP6u4GVXbZxNLAZ2GuUsswDvgbcTg7QU9rep+XAv5X38RrgmX3s37HAv5bhC4GXdJjnNOAjwFLgjLEcv7L8zcDRbeO2Bd4L/Locq/OBOWXarLKft5Xz9QfAkxvLfoUcsN8HNpVzcC7w2TL/NcCBfZTrNcDnyvDFwJFd5rsIOG0M+7tXORdm95jn1cDV5b36FXBUGb8P8B3yZ/R64ITGMmcBF5T3aSPw+l7HsY9yngJc1ng9B/hfYN8O8z69HOvZjXFXAot7baPfJufxpeDnAy+KiN0a074AnJRSejRwAHBZGX8q+cP9WGA38ges9XdWN5JrfDuRa0TLSnr3LaV0LbAEuCLlq3Dv5O7seuD+0lR7cUQ8po/1nwU8CTiIfMXdkxz4LbuTw3xP4ATg3Ih4MkBEvDYiru5WmIjYFvg0cDIPHquHzNJh+IAe05vTmo4GfpJS+m2PsmwD/CdwVdmXo4C3RsSLGrO9nPxhnwN8s5S92/reHxF3k2uVx5XhvwW+HBF3l30nIh4HvAn4YLd1jdPbgRcCzyMHwF/ItdqWi4H9ye/fdcCX2pY/rqxjV3IArgZ+COxCDoSPdNtwRJxZ9ncpcEIZfinwHxFxx0R3DNgArAcuiIhjImJu2/aPAM4F/oH8mTsK+E2Z/FVyzXAP4LXA2RHx3MbiryIfi53Iod/zOJYujFd2KefTyOcTACmlu8kVg6d1mff6lGvyLVd1mfdBfaTq80qhdy2vrwPe1ph+C3ASsGPbch8knyRP6GMba4FjGlf+UWtoZXgxjVpMl3UvpksNrUx/KvlE+y3wf+QP5m6d1k8OiU2UWlEZ9xzgpjK8sKxjh8b05cB7+7yCvY0Hr+Dt2z6afEV8OjAb+BfgAeA1Zfoy4OvAo8lBeyPw5y7bOQ/4SuP1zuSaxj3AfWXcs4Fb2pZ7Fw/Wrk4HLmlMmw9sHmX/ZpGb0ruRa6Lf7jDPxcBxZXgpg6uh3QQ8t/F6P+BPQHRYfvdybLcvr78CfKox/R3AzxuvDwE2jFKmR5KDYxfgSOBrPeYdUw2tLPM4cmvhJnKr4lJgvzLtS8CZHZZ5IrkV0qwFnQ2cU4bPAr433uPYYXvnA6e3jfsZ8Hcd5n0zcHnbuI+1ytbtp58a2gllp1pXkgvKuJZXkZud6yPihxHxnDL+n4EbgO9FxK+bnZwRcXxErC1X5rvJNYld+yjLwKWUrk0pLU4p7VXKMQ/4RJfZH0vup/pZo+wryviWu1JKmxqv15d19lQ6+E8B/rFLOS8B3k++St5cfjaSg5iy7GZyc+Jick2oWw3sD+Qrcmvdd6ZcAz2Y3KSG/AGZ19rPsq/vJodRy4bG8J+A7Tv1s0TEQWX5u8hh+0tys25hWfcry3wvAx6dUrqwS7nHJSIC2Bv4TmNffk7uQ94lImZFxMfKefpH8kU7yOHTcltjeHOH14/qsu1Dy/buJH/4bwS+C4yUsgykzzKltD6ltCSltB/w+DL6i+X33mW77eYBt6eH1oLWk2vkLa2a3KjHsY9i3gvs2DZuR/J5PJF5t+jZyRcRs4FFwLYR0Tp5twPmRMQzUkpXpZSuBI4pHbknk2ske6eUNpKbnadGxAHAZRFxJTnkPk+u9l6RUro/ItbyYHNpEzk0WnbvUcSB/quQlNJ1kb/icVKX9d9BPnmfllK6tctqHhMROzRCbR/gF31s/lnkkFmXzxtmA7PLcd8zpXR/SukzwGcAIuJJwHta604p3QlsuZsXER8CftplW5cCfx8Re6Xuzc7fkGueT+yj7D2llNaSz5nTyFfyMyPi58CrU0o3NGY9Cnhm41zbidwlcGBK6ZgJbD9FxK3AK1NKP2ufHhFvBl4APJ/c4tgN+D0PbcKPd9uryft+OnBPSunsiLgWeEGPYz/Rba6PiM8BnyujfkNuTrf7HfDYiJjdCLV9yH3hW1bXWG/P49iHa8g3ogCIiJ3IF85rusz7pIjYPqV0Xxn3DHK/ZVej1dBeQa6+zif3GR1EbqKtBI6PfOv/dRGxU8p3bv5IrqoTES+NiCeUVL+nrOcBYAfyQbq9zPdGHtrXsxY4IiL2KTv8rh7luw3YKyIeOcp+dBQRT4mIU1u3jssduNeQ+0cetv6U0gPkMD671U8REXu29SsBfKAcm8PJfSVf7aM43yU3r1vH+X3kq99BJfS3j4gDItuH3CfyyZTSXaUc+0fELhGxbUS8GHgL+c7Qw6SUvkeuIX0jIp5dyvoI4NDGbD8FNkbEOyN/xWLbsv1D+tiXbg4G1pTjOa8tzCB3Nrf6Jw8iN/8/T75BQkQsjIjxXsTOAc4q7zERMbfUCCE30+8j11x3oMtxm6DWvu9A7p55WJhF/jrH9uTP5azynm9Tpj0l8tdYHnaBj4jdIn8F5/Hl/JhL7rJoncfnASdFxBERsU1E7F0uiDcA/w2cERHbRcQCcuur153HXsdxNBcBh0TEy8p+fgBYlVK6uX3GlNLV5NbGe0vZjiPX7i/utYHRAu0Ecp/JLSmlDa0fcudvqzbwBuDmUlVf0hj/ROASctXxCuCzKaUfpJTWkdvCV5AD40DyHdTWjnyffFflanL7+ls9yncZOck3xPg6VzeS+4p+EhGbyCfAL8g1y27rfyf5RFhd9vkS4MmNdW4gN61+R+4zWJJSug6ghH+nqxEppT+3HeN7gL+UYYDtyc39e8lhcwU5AFoOJp+cG4EzyXdXO26rOJZ8bJeR+89uIr93LyrluZ8cxgeVaXeQPxg79VjnaFpfVTiQDrXWlNLGtmOwGdhUap+Qmzurxrntj5Dfq8siYmNZz4Iy7QvkC+wG8jEcxncPF1AuUOSLdif/Tt7nY4F/KsOLyrS9yR/w2zssdx/583Y5+f2/inwOngiQUlpJ/mx+lnxeXUq+w53Idz/nk/f9QuAdKaVe+9/rOBIRN0bEqzotmFL6HfnmysfJTfADyHdOW8sujYhmd8+rgSPI5+f7yd+GuKtH2XJHXs0iYjGwMKW0eBK2tZB8Q6PrlwU1fhFxHvDVlNJ/TXVZJlvkLxj/KqXUfvdVDRP6opw0mVJKff/JW21SSu+Z6jLMBFtDoK0lV1klVa76JqekrceM+28bktTN1tDknHQT+GqBtkIppQl/302ZNTRJ1TDQJFXDQJNUDQNNUjUMNEnVMNAkVcNAk1QNA01SNQw0SdUw0CRVw0CTVA0DTVI1DDRJ1TDQJFXDQJNUDQNNUjUMNEnVMNAkVcNAk1QNA01SNQw0SdUw0CRVw0CTVA0DTVI1DDRJ1TDQJFXDQJNUDQNNUjUMNEnVMNAkVcNAk1QNA01SNQw0SdWYNdUF0PS17vWv3zI8f9myaTtNarGGpo5aAdIKj2agTKdpUpOBJqkaBpqkahhokqphoEmqRqSUproM1YmIKg7qdLqTWfNdzpRSTHUZamGgDUEtgabJYaANjk1OSdUw0CRVw0CTVA0DTVI1/FtOVW/1yMiW4UNXrJjCkmjYvMs5BN7lnForP3F43/Me/taVQyxJf7zLOTgG2hAYaJNv9ze9fMLr2PDFbw6gJGNnoA2OgTYEBtrkWTKyauDrPGfFYQNfZy8G2uAYaENgoA3fqpOHHzqHfXrwYdmJgTY43hTQjNIeZK3QWXXyYQ8ZHvQ0zQwGmmakTiHUKZAmMk0zj03OIbDJOTzD6DPrZTL602xyDo6BNgQG2vDt/orTh76NDd8Y/jbAQBskm5yamXZeM9Ul0DRkDW0IrKFNnn6+RNv88uxY558M1tAGx7/llFQNa2hDYA1t8jX/XrOl199tjnX+YbKGNjgG2hAYaFOjFVJjCabVIyNT/gfrBtrgGGhDYKBpLAy0wbEPTVI1DDRJ1TDQJFXDQJNUDQNNUjUMNEnVMNAkVcNAk1QNA01SNQw0SdUw0CRVw0CTVA0DTVI1DDRJ1TDQJFXDQJNUDQNNUjV8jN1WZske524ZPuf3bxnzdGk6s4YmqRoG2lZkyR7nbql1nfP7t2ypjS3Z49yuNbPmeGm68yEpQzCdH5LSCqhWaHUKMpudk8uHpAyOgTYE0znQVixYwMiaNaxYsKDrPK3pzd8aHgNtcGxyysBSNayhDcF0rqG1P4y3+QTxXuM0PNbQBsdAG4LpHGiafgy0wbHJKakaBpqkahhokqphoEmqhoEmqRoGmqRqGGiSqmGgSaqGgSapGgaapGoYaJKqYaBJqoaBJqkaBpqkavjUpxmq17/J9l9oa2tlDU1SNQy0GajX05tafHKTtkb+x9ohmIz/WNvt6U0+uWnm8T/WDo6BNgSTEWijPb3JJzfNHAba4NjkrIihpa2dgTYDrR4ZYc7cuQDMmTuXkTVrtrxuDa8eGWFkzZqHzCvVzibnEPjUJ42FTc7BsYYmqRoGmqRqGGiSqmGgSaqGgSapGgaapGoYaJKqYaBJqoaBJqkaBpqkahhokqphoEmqhoEmqRoGmqRqGGgauCV7nNv1OQa9pkkTZaBJqoaBpoFqPpGq+TSqTg9tsaamQfNBwxqYVpg1n0DVHlrt06VB8l9wD8HW+i+4258y1XoiVfvTqVoPc/FpVJn/gntwrKFpYJoPblk9MsKhK1awemSE1SMjABy6YgXAw15Lg2IfmgaqU1g1h1vT24elQbDJOQRba5NT42OTc3CsoUmqhoEmqRoGmqRqGGiSqmGgSaqGgSapGgaapGoYaJKqYaBJqoaBJqkaBpqkahhokqphoEmqhoEmqRoGmqRqGGiSqmGgSaqGgSapGgaapGoYaJKqYaBJqoaBJqkaBpqkahhokqphoEmqhoEmqRoGmqRqGGiSqmGgSaqGgSapGgaapGoYaJKqYaBJqoaBJqkaBpqkahhokqphoEmqhoEmqRoGmqRqGGiSqmGgSaqGgSapGgaapGoYaJKqYaBJqoaBJqkaBpqkahhokqphoEmqhoEmqRoGmqRqGGiSqmGgSaqGgSapGgaapGoYaJKqYaBJqoaBJvUwsnSEkaUjU10M9SlSSlNdhupEhAe1MiNLR1ixeMVQ1p1SiqGseCtkoA2BgVaHdevWMX/+/C3DwJbXg2SgDY6BNgQG2szXCrBOBh1qBtrgzJrqAkjT0a+XvXaqi6Bx8KaA1MFLP7R2y+/WcHO8picDTeqiPbwMs+nPPrQhsA9NY2Ef2uDYhzYEp5x35FQXQdoq2eSUVA2bnENgk1NjYZNzcKyhSaqGgSapGt4UmIaWL1/OokWLpv1yk2XVyYd1nXbYp1dNYkk03Rlo08jy5csBWLRo0UOGp9ty0nTlTYEhGM9NgVagdNIrZCZ7ualQew3NmwKDYx+apGoYaNNEs1bUbXg6LCdNZwbaNNIeJv2Gy2QvJ01X9qENwXi/WNupX2ssnfuTtdxksw9N/TLQhsC/FNBYGGiDY5NTUjUMNEnVMNAkVcNAk1QNA01SNQw0SdUw0CRVw0CTVA0DTVI1DDRJ1fBPnyRVwxqapGoYaJKqYaBJqoaBJqkaBpqkahhokqrx/20k7fLSWqbsAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import time # for tracking time\n",
    "\n",
    "tStart = time.time()\n",
    "\n",
    "# first create an instance of the TpgTrainer\n",
    "# this creates the whole population and everything\n",
    "# teamPopSize should realistically be at-least 100\n",
    "trainer = Trainer(actions=env.action_space.n, teamPopSize=20) \n",
    "\n",
    "curScores = [] # hold scores in a generation\n",
    "summaryScores = [] # record score summaries for each gen (min, max, avg)\n",
    "\n",
    "# 5 generations isn't much (not even close), but some improvements\n",
    "# should be seen.\n",
    "for gen in range(5): # generation loop\n",
    "    curScores = [] # new list per gen\n",
    "    \n",
    "    agents = trainer.getAgents()\n",
    "    \n",
    "    while True: # loop to go through agents\n",
    "        teamNum = len(agents)\n",
    "        agent = agents.pop()\n",
    "        if agent is None:\n",
    "            break # no more agents, so proceed to next gen\n",
    "        \n",
    "        state = env.reset() # get initial state and prep environment\n",
    "        score = 0\n",
    "        for i in range(500): # run episodes that last 500 frames\n",
    "            show_state(env, i, 'Assault', 'Gen #' + str(gen) + \n",
    "                       ', Team #' + str(teamNum) +\n",
    "                       ', Score: ' + str(score)) # render env\n",
    "            \n",
    "            # get action from agent\n",
    "            # must transform to at-least int-32 (for my getState to bitshift correctly)\n",
    "            act = agent.act(getState(np.array(state, dtype=np.int32))) \n",
    "\n",
    "            # feedback from env\n",
    "            state, reward, isDone, debug = env.step(act)\n",
    "            score += reward # accumulate reward in score\n",
    "            if isDone:\n",
    "                break # end early if losing state\n",
    "\n",
    "        agent.reward(score) # must reward agent (if didn't already score)\n",
    "            \n",
    "        curScores.append(score) # store score\n",
    "        \n",
    "        if len(agents) == 0:\n",
    "            break\n",
    "            \n",
    "    # at end of generation, make summary of scores\n",
    "    summaryScores.append((min(curScores), max(curScores),\n",
    "                    sum(curScores)/len(curScores))) # min, max, avg\n",
    "    trainer.evolve()\n",
    "    \n",
    "#clear_output(wait=True)\n",
    "print('Time Taken (Hours): ' + str((time.time() - tStart)/3600))\n",
    "print('Results:\\nMin, Max, Avg')\n",
    "for result in summaryScores:\n",
    "    print(result[0],result[1],result[2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Example: Generational Selection with Multiprocessing (no graphics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This example is just to show a different way to run the API, a far superior way. It uses a different method to get the agents, doesn't use graphics (but can), and uses multiprocessing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Run each agent in this method for parallization.\n",
    "Args:\n",
    "    args: (TpgAgent, envName, scoreList, numEpisodes, numFrames)\n",
    "\"\"\"\n",
    "def runAgent(args):\n",
    "    agent = args[0]\n",
    "    envName = args[1]\n",
    "    scoreList = args[2]\n",
    "    numEpisodes = args[3] # number of times to repeat game\n",
    "    numFrames = args[4] \n",
    "    \n",
    "    # skip if task already done by agent\n",
    "    if agent.taskDone(envName):\n",
    "        print('Agent #' + str(agent.agentNum) + ' can skip.')\n",
    "        scoreList.append((agent.team.id, agent.team.outcomes))\n",
    "        return\n",
    "    \n",
    "    env = gym.make(envName)\n",
    "    valActs = range(env.action_space.n) # valid actions, some envs are less\n",
    "    \n",
    "    scoreTotal = 0 # score accumulates over all episodes\n",
    "    for ep in range(numEpisodes): # episode loop\n",
    "        state = env.reset()\n",
    "        scoreEp = 0\n",
    "        numRandFrames = 0\n",
    "        if numEpisodes > 1:\n",
    "            numRandFrames = random.randint(0,30)\n",
    "        for i in range(numFrames): # frame loop\n",
    "            if i < numRandFrames:\n",
    "                env.step(env.action_space.sample())\n",
    "                continue\n",
    "\n",
    "            act = agent.act(getState(np.array(state, dtype=np.int32)))\n",
    "\n",
    "            # feedback from env\n",
    "            state, reward, isDone, debug = env.step(act)\n",
    "            scoreEp += reward # accumulate reward in score\n",
    "            if isDone:\n",
    "                break # end early if losing state\n",
    "                \n",
    "        print('Agent #' + str(agent.agentNum) + \n",
    "              ' | Ep #' + str(ep) + ' | Score: ' + str(scoreEp))\n",
    "        scoreTotal += scoreEp\n",
    "       \n",
    "    scoreTotal /= numEpisodes\n",
    "    env.close()\n",
    "    agent.reward(scoreTotal, envName)\n",
    "    scoreList.append((agent.team.id, agent.team.outcomes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time Taken (Hours): 0.6896040613121457\n",
      "Results:\n",
      "Min, Max, Avg\n",
      "-100.0 3.0 -45.52777777777778\n",
      "-100.0 15.0 -29.719444444444445\n",
      "-100.0 21.0 -25.533333333333335\n",
      "-100.0 36.0 -25.680555555555557\n",
      "-100.0 36.0 -21.508333333333333\n",
      "-100.0 36.0 -20.358333333333334\n",
      "-100.0 36.0 -18.81388888888889\n",
      "-100.0 36.0 -18.332409972299168\n",
      "-100.0 36.0 -14.911357340720222\n",
      "-100.0 36.0 -16.28808864265928\n",
      "-100.0 36.0 -14.376731301939058\n",
      "-100.0 36.0 -13.520775623268698\n",
      "-100.0 36.0 -15.57617728531856\n",
      "-100.0 36.0 -12.426592797783934\n",
      "-100.0 36.0 -13.578947368421053\n",
      "-100.0 36.0 -11.520775623268698\n",
      "-100.0 36.0 -9.986149584487535\n",
      "-100.0 36.0 -12.354570637119114\n",
      "-100.0 36.0 -13.861495844875346\n",
      "-100.0 36.0 -11.357340720221607\n",
      "-100.0 37.0 -11.734072022160666\n",
      "-100.0 37.0 -9.847222222222221\n",
      "-100.0 37.0 -11.986149584487535\n",
      "-100.0 37.0 -8.13888888888889\n",
      "-100.0 42.0 -8.894736842105264\n",
      "-100.0 42.0 -11.602777777777778\n",
      "-100.0 42.0 -7.044444444444444\n",
      "-100.0 42.0 -8.502777777777778\n",
      "-100.0 42.0 -7.719444444444444\n",
      "-100.0 42.0 -7.124653739612189\n",
      "-100.0 42.0 -6.719444444444444\n",
      "-100.0 42.0 -5.914127423822714\n",
      "-100.0 42.0 -8.355555555555556\n",
      "-100.0 42.0 -2.6\n",
      "-100.0 42.0 -3.1305555555555555\n",
      "-100.0 42.0 -7.230555555555555\n",
      "-100.0 52.0 -6.538888888888889\n",
      "-100.0 52.0 -3.3666666666666667\n",
      "-100.0 52.0 -5.663888888888889\n",
      "-100.0 52.0 -2.221606648199446\n",
      "-100.0 52.0 -2.352777777777778\n",
      "-100.0 52.0 -3.736842105263158\n",
      "-100.0 52.0 -1.2694444444444444\n",
      "-100.0 52.0 -3.2416666666666667\n",
      "-100.0 52.0 -1.6888888888888889\n",
      "-100.0 52.0 -5.263888888888889\n",
      "-100.0 52.0 3.9113573407202216\n",
      "-100.0 52.0 -6.102777777777778\n",
      "-100.0 52.0 -0.26944444444444443\n",
      "-100.0 52.0 2.9058171745152355\n",
      "-100.0 52.0 0.43213296398891965\n",
      "-100.0 52.0 -2.013888888888889\n",
      "-100.0 64.0 0.2638888888888889\n",
      "-100.0 64.0 0.9916666666666667\n",
      "-100.0 64.0 2.7416666666666667\n",
      "-100.0 64.0 0.044444444444444446\n",
      "-100.0 64.0 1.4805555555555556\n",
      "-100.0 64.0 0.28888888888888886\n",
      "-100.0 64.0 3.6777777777777776\n",
      "-100.0 64.0 -0.6398891966759003\n",
      "-100.0 64.0 2.763888888888889\n",
      "-100.0 64.0 1.9388888888888889\n",
      "-100.0 64.0 3.713888888888889\n",
      "-100.0 64.0 -1.0858725761772854\n",
      "-100.0 64.0 3.3583333333333334\n",
      "-100.0 67.0 3.561111111111111\n",
      "-100.0 67.0 5.413888888888889\n",
      "-100.0 67.0 4.2444444444444445\n",
      "-100.0 67.0 3.2666666666666666\n",
      "-100.0 67.0 2.0027777777777778\n",
      "-100.0 67.0 9.294444444444444\n",
      "-100.0 67.0 0.08888888888888889\n",
      "-100.0 67.0 9.277777777777779\n",
      "-100.0 67.0 0.019444444444444445\n",
      "-100.0 67.0 0.5611111111111111\n",
      "-100.0 67.0 7.3\n",
      "-100.0 67.0 7.308333333333334\n",
      "-100.0 67.0 0.7361111111111112\n",
      "-100.0 67.0 8.33611111111111\n",
      "-100.0 67.0 4.980555555555555\n",
      "-100.0 67.0 5.222222222222222\n",
      "-100.0 67.0 8.027777777777779\n",
      "-100.0 67.0 4.902777777777778\n",
      "-100.0 67.0 7.706371191135734\n",
      "-100.0 67.0 9.587257617728532\n",
      "-100.0 67.0 5.680555555555555\n",
      "-100.0 67.0 7.705555555555556\n",
      "-100.0 67.0 7.238888888888889\n",
      "-100.0 67.0 9.816666666666666\n",
      "-100.0 67.0 9.430555555555555\n",
      "-100.0 67.0 9.863888888888889\n",
      "-100.0 69.0 2.581717451523546\n",
      "-100.0 69.0 12.730555555555556\n",
      "-100.0 69.0 7.697222222222222\n",
      "-100.0 69.0 10.734072022160666\n",
      "-100.0 69.0 9.481994459833794\n",
      "-100.0 69.0 8.966666666666667\n",
      "-100.0 70.0 11.744444444444444\n",
      "-100.0 70.0 9.058333333333334\n",
      "-100.0 70.0 10.683333333333334\n"
     ]
    }
   ],
   "source": [
    "import multiprocessing as mp\n",
    "import time\n",
    "\n",
    "tStart = time.time()\n",
    "\n",
    "envName = 'Boxing-v0'\n",
    "# get num actions\n",
    "env = gym.make(envName)\n",
    "acts = env.action_space.n\n",
    "del env\n",
    "\n",
    "trainer = Trainer(actions=acts, teamPopSize=360)\n",
    "\n",
    "processes = 23\n",
    "man = mp.Manager()\n",
    "pool = mp.Pool(processes=processes, maxtasksperchild=1)\n",
    "    \n",
    "allScores = [] # track all scores each generation\n",
    "\n",
    "for gen in range(100): # do 100 generations of training\n",
    "    scoreList = man.list()\n",
    "    \n",
    "    # get agents, noRef to not hold reference to trainer in each one\n",
    "    # don't need reference to trainer in multiprocessing\n",
    "    agents = trainer.getAgents() # swap out agents only at start of generation\n",
    "\n",
    "    # run the agents\n",
    "    pool.map(runAgent, \n",
    "        [(agent, envName, scoreList, 1, 18000)\n",
    "        for agent in agents])\n",
    "    \n",
    "    # apply scores, must do this when multiprocessing\n",
    "    # because agents can't refer to trainer\n",
    "    teams = trainer.applyScores(scoreList)\n",
    "    # important to remember to set tasks right, unless not using task names\n",
    "    # task name set in runAgent()\n",
    "    trainer.evolve(tasks=[envName]) # go into next gen\n",
    "    \n",
    "    # an easier way to track stats than the above example\n",
    "    scoreStats = trainer.fitnessStats\n",
    "    allScores.append((scoreStats['min'], scoreStats['max'], scoreStats['average']))\n",
    "    \n",
    "    clear_output()\n",
    "    print('Time Taken (Hours): ' + str((time.time() - tStart)/3600))\n",
    "    print('Gen: ' + str(gen))\n",
    "    print('Results so far: ' + str(allScores))\n",
    "    \n",
    "clear_output()\n",
    "print('Time Taken (Hours): ' + str((time.time() - tStart)/3600))\n",
    "print('Results:\\nMin, Max, Avg')\n",
    "for score in allScores:\n",
    "    print(score[0],score[1],score[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:oaigym]",
   "language": "python",
   "name": "conda-env-oaigym-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}