Navigation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:unityagents:\n",
      "'Academy' started successfully!\n",
      "Unity Academy name: Academy\n",
      "        Number of Brains: 1\n",
      "        Number of External Brains : 1\n",
      "        Lesson number : 0\n",
      "        Reset Parameters :\n",
      "\t\t\n",
      "Unity brain name: BananaBrain\n",
      "        Number of Visual Observations (per agent): 0\n",
      "        Vector Observation space type: continuous\n",
      "        Vector Observation space size (per agent): 37\n",
      "        Number of stacked Vector Observation: 1\n",
      "        Vector Action space type: discrete\n",
      "        Vector Action space size (per agent): 4\n",
      "        Vector Action descriptions: , , , \n"
     ]
    }
   ],
   "source": [
    "# Import packages\n",
    "from unityagents import UnityEnvironment\n",
    "import numpy as np\n",
    "import random\n",
    "import torch\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "# Set enviroment\n",
    "env = UnityEnvironment(file_name=\"/media/riley/Work/PersonalProjects/AI/Deep_Reinforcement_Learning/Project1/Banana_Linux/Banana.x86_64\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the default brain\n",
    "brain_name = env.brain_names[0]\n",
    "brain = env.brains[brain_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of agents: 1\n",
      "Number of actions: 4\n",
      "States look like: [1.         0.         0.         0.         0.84408134 0.\n",
      " 0.         1.         0.         0.0748472  0.         1.\n",
      " 0.         0.         0.25755    1.         0.         0.\n",
      " 0.         0.74177343 0.         1.         0.         0.\n",
      " 0.25854847 0.         0.         1.         0.         0.09355672\n",
      " 0.         1.         0.         0.         0.31969345 0.\n",
      " 0.        ]\n",
      "States have length: 37\n"
     ]
    }
   ],
   "source": [
    "# reset the environment\n",
    "env_info = env.reset(train_mode=True)[brain_name]\n",
    "\n",
    "# number of agents in the environment\n",
    "print('Number of agents:', len(env_info.agents))\n",
    "\n",
    "# number of actions\n",
    "action_size = brain.vector_action_space_size\n",
    "print('Number of actions:', action_size)\n",
    "\n",
    "# examine the state space \n",
    "state = env_info.vector_observations[0]\n",
    "print('States look like:', state)\n",
    "state_size = len(state)\n",
    "print('States have length:', state_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DQN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "#from dqn_agent import Agent\n",
    "from agent import DQNPRBAgent\n",
    "from agent import DQNAgent\n",
    "\n",
    "agent = DQNAgent(state_size=37, action_size=4, seed=0, use_dueling=True, use_double=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 100\tAverage Score: 0.67\n",
      "Episode 200\tAverage Score: 3.34\n",
      "Episode 300\tAverage Score: 6.08\n",
      "Episode 400\tAverage Score: 8.28\n",
      "Episode 500\tAverage Score: 12.38\n",
      "Episode 539\tAverage Score: 13.04\n",
      "Environment solved in 439 episodes!\tAverage Score: 13.04\n",
      "\tModel Saved\n",
      "Episode 543\tAverage Score: 13.09\tModel Saved\n",
      "Episode 544\tAverage Score: 13.16\tModel Saved\n",
      "Episode 545\tAverage Score: 13.25\tModel Saved\n",
      "Episode 547\tAverage Score: 13.34\tModel Saved\n",
      "Episode 548\tAverage Score: 13.36\tModel Saved\n",
      "Episode 552\tAverage Score: 13.43\tModel Saved\n",
      "Episode 553\tAverage Score: 13.45\tModel Saved\n",
      "Episode 554\tAverage Score: 13.47\tModel Saved\n",
      "Episode 555\tAverage Score: 13.49\tModel Saved\n",
      "Episode 559\tAverage Score: 13.52\tModel Saved\n",
      "Episode 560\tAverage Score: 13.54\tModel Saved\n",
      "Episode 561\tAverage Score: 13.57\tModel Saved\n",
      "Episode 564\tAverage Score: 13.57\tModel Saved\n",
      "Episode 570\tAverage Score: 13.63\tModel Saved\n",
      "Episode 571\tAverage Score: 13.69\tModel Saved\n",
      "Episode 572\tAverage Score: 13.71\tModel Saved\n",
      "Episode 574\tAverage Score: 13.77\tModel Saved\n",
      "Episode 575\tAverage Score: 13.82\tModel Saved\n",
      "Episode 577\tAverage Score: 13.83\tModel Saved\n",
      "Episode 578\tAverage Score: 13.86\tModel Saved\n",
      "Episode 587\tAverage Score: 13.90\tModel Saved\n",
      "Episode 590\tAverage Score: 13.90\tModel Saved\n",
      "Episode 593\tAverage Score: 13.92\tModel Saved\n",
      "Episode 594\tAverage Score: 13.99\tModel Saved\n",
      "Episode 595\tAverage Score: 14.05\tModel Saved\n",
      "Episode 600\tAverage Score: 14.00\n",
      "Episode 603\tAverage Score: 14.11\tModel Saved\n",
      "Episode 604\tAverage Score: 14.22\tModel Saved\n",
      "Episode 605\tAverage Score: 14.24\tModel Saved\n",
      "Episode 606\tAverage Score: 14.36\tModel Saved\n",
      "Episode 607\tAverage Score: 14.36\tModel Saved\n",
      "Episode 608\tAverage Score: 14.42\tModel Saved\n",
      "Episode 615\tAverage Score: 14.47\tModel Saved\n",
      "Episode 616\tAverage Score: 14.59\tModel Saved\n",
      "Episode 617\tAverage Score: 14.64\tModel Saved\n",
      "Episode 629\tAverage Score: 14.67\tModel Saved\n",
      "Episode 631\tAverage Score: 14.69\tModel Saved\n",
      "Episode 700\tAverage Score: 14.34\n",
      "Episode 776\tAverage Score: 14.69\tModel Saved\n",
      "Episode 778\tAverage Score: 14.72\tModel Saved\n",
      "Episode 779\tAverage Score: 14.73\tModel Saved\n",
      "Episode 780\tAverage Score: 14.76\tModel Saved\n",
      "Episode 786\tAverage Score: 14.79\tModel Saved\n",
      "Episode 800\tAverage Score: 14.63\n",
      "Episode 807\tAverage Score: 14.81\tModel Saved\n",
      "Episode 900\tAverage Score: 13.43\n",
      "Episode 998\tAverage Score: 14.82\tModel Saved\n",
      "Episode 999\tAverage Score: 14.90\tModel Saved\n",
      "Episode 1000\tAverage Score: 14.87\n",
      "Episode 1001\tAverage Score: 15.02\tModel Saved\n",
      "Episode 1100\tAverage Score: 14.08\n",
      "Episode 1130\tAverage Score: 15.03\tModel Saved\n",
      "Episode 1137\tAverage Score: 15.03\tModel Saved\n",
      "Episode 1138\tAverage Score: 15.06\tModel Saved\n",
      "Episode 1141\tAverage Score: 15.06\tModel Saved\n",
      "Episode 1146\tAverage Score: 15.08\tModel Saved\n",
      "Episode 1147\tAverage Score: 15.14\tModel Saved\n",
      "Episode 1149\tAverage Score: 15.26\tModel Saved\n",
      "Episode 1150\tAverage Score: 15.28\tModel Saved\n",
      "Episode 1151\tAverage Score: 15.37\tModel Saved\n",
      "Episode 1159\tAverage Score: 15.45\tModel Saved\n",
      "Episode 1160\tAverage Score: 15.52\tModel Saved\n",
      "Episode 1172\tAverage Score: 15.59\tModel Saved\n",
      "Episode 1173\tAverage Score: 15.59\tModel Saved\n",
      "Episode 1174\tAverage Score: 15.59\tModel Saved\n",
      "Episode 1175\tAverage Score: 15.64\tModel Saved\n",
      "Episode 1180\tAverage Score: 15.66\tModel Saved\n",
      "Episode 1181\tAverage Score: 15.77\tModel Saved\n",
      "Episode 1182\tAverage Score: 15.85\tModel Saved\n",
      "Episode 1200\tAverage Score: 15.74\n",
      "Episode 1300\tAverage Score: 14.49\n",
      "Episode 1400\tAverage Score: 14.77\n",
      "Episode 1469\tAverage Score: 15.95\tModel Saved\n",
      "Episode 1470\tAverage Score: 16.01\tModel Saved\n",
      "Episode 1471\tAverage Score: 16.02\tModel Saved\n",
      "Episode 1472\tAverage Score: 16.02\tModel Saved\n",
      "Episode 1500\tAverage Score: 15.48\n",
      "Episode 1600\tAverage Score: 15.37\n",
      "Episode 1700\tAverage Score: 14.78\n",
      "Episode 1800\tAverage Score: 15.12\n",
      "Episode 1842\tAverage Score: 16.04\tModel Saved\n",
      "Episode 1843\tAverage Score: 16.07\tModel Saved\n",
      "Episode 1844\tAverage Score: 16.12\tModel Saved\n",
      "Episode 1845\tAverage Score: 16.16\tModel Saved\n",
      "Episode 1846\tAverage Score: 16.21\tModel Saved\n",
      "Episode 1900\tAverage Score: 15.72\n",
      "Episode 2000\tAverage Score: 15.03\n",
      "Episode 2100\tAverage Score: 14.72\n",
      "Episode 2200\tAverage Score: 14.56\n",
      "Episode 2300\tAverage Score: 14.89\n",
      "Episode 2400\tAverage Score: 14.74\n",
      "Episode 2500\tAverage Score: 14.96\n",
      "Episode 2600\tAverage Score: 14.87\n",
      "Episode 2700\tAverage Score: 14.43\n",
      "Episode 2800\tAverage Score: 13.32\n",
      "Episode 2900\tAverage Score: 15.03\n",
      "Episode 3000\tAverage Score: 15.91\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXecFdX5/z/P7rL0vksHlyZFQcBVFEFBLAh2Eks0ITb0a4n5qUnAghpLSLEkMbYk2AOxYEEQBSUWBBGQjvSl16WXZdv5/TEzd+fOnXqn3/u8Xy/YO2fKec6ZM+c55TnPISEEGIZhmOwlJ2wBGIZhmHBhRcAwDJPlsCJgGIbJclgRMAzDZDmsCBiGYbIcVgQMwzBZDisChmGYLIcVAcMwTJbDioBhGCbLyQtbADsUFBSIoqKisMVgGIaJFQsWLNgjhCi0ui4WiqCoqAjz588PWwyGYZhYQUQb7VzHQ0MMwzBZDisChmGYLIcVAcMwTJbDioBhGCbLYUXAMAyT5bAiYBiGyXJYETAMw2Q5rAgYJsIcKqvAh4u2hi0Gk+HEYkEZw2QrYyYvxdQl29G5sAFObts4bHGYDIV7BAwTYXYcKAMAlFVUhSwJk8mwImCYGCDCFoDJaFgRMEyEobAFYLICVgQMwzBZDisChmGYLIcVAcPY5Lkv1uDql+aELUbW8s78zRj851lhi5Hg2pfn4m+frwlbDE9g81GGsclfPlsdtghZzW/eXRK2CEnMWV+KOetL8auhXcMWxTXcI2AYhslyWBEwTAwQbD+aQHBmeI5vioCI2hPRLCJaQUTLiehuOfwRItpKRIvkf8P9koFh4g6x/SgTAH7OEVQCuFcIsZCIGgJYQEQz5HPPCCH+4mPcDMNkKEKwgvQa33oEQojtQoiF8u9DAFYCaOtXfAyTjazbfRiHj1f69vyt+49hz+Hjhuc3lh7BgaMVvsWvh52BobKKKqzacciT+CqrqrFs6wFPnqXH0i0HdIe79h0px9Ql27HrUJlvcSsEMkdAREUA+gL4Tg66k4iWENEEImpqcM9oIppPRPN3794dhJgME1mMxsWHPvUlrv/Xd7rnvOCs8V+g+PGZhufP+fP/cOGzX/kWf7r89t0luPDZr7D/aLnrZz0zczUu/vs3WLn9oAeSJTNjxU5c8tw3eHv+5pRzG0qP4I7/LMSKbd7Hq8V3RUBEDQC8B+DXQoiDAF4A0BlAHwDbATyld58Q4mUhRLEQoriwsNBvMRkmkpANJxOLNu8PQBJjdhz0v8Wqxs5k8bwNewEAR8vdO+tbskXqDew6ZNwzSpeSPUcAAKt3Hk45F+ScuK+KgIhqQVICbwkhJgOAEGKnEKJKCFEN4J8ATvdTBoaJM4LdzaXgJEeiPpdgLp+Qr/E/EX5aDRGAfwNYKYR4WhXeWnXZFQCW+SUDwzCMV/hptmr26CB0mZ9WQ2cB+DmApUS0SA67H8C1RNQHkrorAXCrjzIwTKyxMzSUbdipj73sSQXRItcjyKEh3xSBEOIb6CuzaX7FyTCZCg8Q1eCkko+LItVLkxIShB7ilcUZxn+/34SPl2zz/LnPzFiNhZv2WV5Xevg4xry3JGVHrYWb9uGZGen76tlUehTjPlyGqmqB309ZgbW7vDENtKKiqhpjJy9N7BTGeIcQAo9OWY51u1MnSu2wYONe/HVmZjh902PRJskIIAhlxoogw/jde0tx539+8Py5f/18Da58/lvL656c9iMmfb8ZHy/ZnhR+5fPf4q8uPDX+atIPeH3ORnyybDsmzN6AURO+T/tZTvhq9W5MnLcJD7y/NJD4somS0qN4ZXYJbnrV2btUhkxGvjAHz8yMtyNAZdhJbxjoiWkrA5ODFQHjKUoX1+uJNeVpYbmZCW1oJh4jG5EjLu6I7LxeHhpiGBnlW4jJ9804wM936mUlGlbZC6ItwIqAiQXKB82eJzOHdCs4LgLew4qA8RRlYsvrb5VHSDIXPyr2uOgKWz0WHhpizDha7o+zMTvPPVZeFbnWuSJ3WUUVqqsFjldW4VCZfYdo2nQfK6/C8cpq2/dXVlWnWEupKa+sRkVVtWX+VlRVo1wTb1Sy2ui9OykPSvrNKsHjlVWorNLP+8rq5PBjJm4kjKKorKrG8Upz9xMJOU2vSp/KKqk8AOY9XbYaYgxZu+sQeo77FJMXbvH0ud+u24Oe4z7Ft2v3GF6z40AZeoybjgmzSzyN2w2rd0r5MWneJnR/aDr+OP1HdHtwOno98pmt+9+Zvxk9x32aMGXcf7QcPcZNx+1vLbQtwy9f+R7dH5pueL7bQ5+g6wOfoOe4TzF3fanhdac/MRMnP/yp7XiD4mBZBXqMm45ndEw2e4ybjr9/sdbyGR8u2oqe4z5N8gyqZ0Pf7cHpuPafc3WfoXWy12PcdBw45swD6lUvzUG3B43f1bSl29Fz3Ke+eh294vlv8eS0H317vhNYEcSUldulD+nzH3d5+lzFWZdZRbVl31EAwFSd9Qp+WTiYmdkBSHiG/GixJNM7C5wpyBkrdgIA1sjOv7Sul+20dr8xUZ7SM2p+zy/Za3jdvqMVKDdoDYfJviOSJ88Pftiqe/6DRfrhaj5fKZXXldsPWrZ0vy/RX7eyeEtq5az1Mmr1uhZuMnfUN0v+rpZv89H9tErJmInLVkOMNR4PGdjphiYmbr2N2jxOq/Oar0VdcdupxCMy8pJCFOdGDFf2OsjEwLw2uIwnCiuT2WqICQ3zb9q8dW7jAWljVAklzEt1TkdlfF2NU/81ui4IAk6YVaXoRBq16FF8P7oiRVBOr2BFwCRhp34y6xH41XqpMR91fm91WjVN+C3BuOFUMfnbK3BXayeSElAxMPU+Gmc31Iy/1FTG/jRTwnaLmxqneS9E+60Ig99xRa81Htoqa/cjQ0nvy890RGFoxw5m3zHPETCWfL1af4JSbU30446DmClPhgLAobIKvD6nRLcFZ1Xmjqj2x1278xA+W74jcTx28lJMkSeQJ8zeYGhKOXHeJuw9UjO5t2Dj3qTJ6R0HyvDegi3436pdhlYbW/cfw6GyCuw9Uo6J8zYlwufoTHKnU9FYfXzrdh/G9GXbdc+VyhPNX67enZDf7S5iHy7aisqqalz10hzdvXiVyeeyiiq8MnsDqqsFpizehk2lR13Fq0bJky37jumWHb2e17b9x3DNy3MSk+/KFR8u2oat+48lXTt92Q6s3XXY1ARX4d/fbDA9X1ktxVRRVY3Plu/Amp3mTgrnri/Fgo1SHpYePo73VN+PuiwoMlZXJ6f1aHklXp29ISlflm09gC9Xp26za1Z2AKln9fqcElN5vYYVQcw5ZLBx+T1vL05U2sOe/Ro3vz4/ce7hj5Zj3IfLMWedsWWQUQvlsY9XJLqqR8qrMPqNBQCAA0crMHHeJpRVSNYuP+44pOsQbM3OQxg7eSnunlTjGG/kC3Nwzcs1poLX/Wsu7n1nMX75yve4+O/fGMr48IfLcfekHzB28lKs333EcVr0sNv6GvrUl7jtTX3T0rsmSmkbNWFeQv7L/zHbtgxqNu+VKvJJ32/GW99twrwNexN7BKtT9ZMX5wAAnp6xGo9OWYGPl27HXRN/wPC/fZ1WvFbMXW9s9aRm4B+/wNz1e1Mcy33x466kdw4At725AOc9/SX+McvaDPWxj1eYnt9/VDInnfDNBox+YwHOf8Z8X+VrXp6LkS9Ieag2GdYWB0XGz1bsSAp/YupKPDJlRcIqCgAu/vs3GDVhXkpcZmUHkBoz4z5cbiiDH7AiiCl2urxGY+OKGeAxnZaXVUW4/2iFbszaRT6ApBy0KAu09hw23lR810GdvWF15iX2Hi3HbnkfWaPFR0DwQyilJmlzinpB22EDpa9GMaM8Ji+GsnNPOuiZt+rls9JwdrLfr9M1AWYcKnOefrXpsHp8Xt2g0C40VBRPmcUiNSPUeXe8IvnZPDTE+IKdySfzySt78egponSX1NdYBdmbJPC68g9rniG9iW7/lV+Ozjsyi1M5Z2dCOexRffX3YSRLjra8uZ2cThLA1aPSghVBBmNVNPW+SS8tFMwrBvcWJnYfkU5lGnZlpKCWPKXy0Z3jCUZyp/GYVZTadIS1NaQT8nL0Gx7+5D9bDTE+YMels9E5IvuFvdqHVqmhXGb3BNyc99KSSy17lOpH/R6BN+kOO512os/RywB4I7v2ETw0xEQOIvsF09MNxPXMJ6FqiZnIZEeKKC5qApBknaJNYpgi67XazRS/ae/QA3mMn+3u6UblKtfjochkhR+8JvRt83omfL5bvxfn92xpes2Cjfsw8oVvMXvMuWjbpG4iXF0wz33qf2nFP3nhVjx9VZ+ksEnzNhte/+rsDXhi2krUrZWbFF40Zioa1ZGLqkqur1Smec+a7F2rbqmu330Y5z71ZeK4bZO6mD3m3KTrpy/bgdveXJAU9r9VqWaARqzeeRinPTEzSX4tf/50FTbvPYrxI3sbPuenL36bZBVmVT+o4/lC44Pq4yXbkrYw7dqiAWbcc07KM8oqqtD9oel49uo+uLxv26Rz6vgNGsQAgNveWIAj5ZV446b+ibBdh47r5oMeb83dZH2RTd6e78zn1Lrdh7FmV80eyuo03/hqjeVdbq5+BigWRyXjRyTClHR/fNdAnNy2se59E+dtwsR5mzCid2tcXdw+6RxbDTGG2Gk0GNkiq+9VbPBnr9lj+FytaaabBssbczcanntkygpUVOk3rQ46tP5I8jWkCp+hWk8BIMWWHQCem+V+Q/TdNqxkJn1vrBSBVKdr2l6RWSv0M006n5mRbMqrruzU7DxYBkAyQ9Vi1WpVWt/Tl+/A12vMHfAZPRfQt0iywqux+c+W77S+COlNFk9eaO2Ub+qS7aEMjbEiyGKEEIafj18rlt2Sjlwieo4808JNBeH123RqNRQXtOWLDFRMyjCdjbRX6ZhY6xHGamhWBFlJ6i5iygdgVQjJ8NMIBruVjV7aAJu+lCJjM5SMq7Fjh5W0vpM7c1nSNQ6IaqPDa6rS1JSx9jVERO2JaBYRrSCi5UR0txzejIhmENEa+W9Tv2TIZNwUDXW5MixjHqwjMCPd1mM6tyVVYJaKLnyrFSNSJ4uDrUCdKtRMwCidZr6tjLA74pVpQ0OVAO4VQvQEcAaAO4ioJ4AxAD4XQnQF8Ll8zISMHesbhbhVAuqPNG6yqwlSdisnd/qKPDta9oCz+RoFrX8i42cHj2+KQAixXQixUP59CMBKAG0BXAbgNfmy1wBc7pcMcWPz3qO6TsLml+zFrFXGO5FVVQs898WalH1utWzTTIwKUeOkrfRIOcorq/G9vEOZUmQ37DH24WOHldsP4mOdncxW7TyEXYfKdO8xmxiebbELmILatcCqHYcSO6/psWDjPhyrkPc7rqzGEp0dsMxYtvUAxn/ibsvBZVsPYNaqXbb3352yeBt+3G7uSE2NXhX0xY87E5PDS7bsx4KNexPuFTbtPZrwcwRIewjPU+2qNnvtnhRZqwWwVjUJreccT1c2k/px+rIdWGrjfSgTzAs27sO0pcYO3ZR8VijRlO/Ne5O/kU2lR3UNCrYdqAkze2dq7A4N7dbsjheEYgjEfJSIigD0BfAdgJZCCOVN7QCga99IRKMBjAaADh06+C9kBBj0p1kAkk3P9h0pTzgUm3z7APTrkDqSdud/FuKTZTvw8RLjDwAABoz/IunZny3fkSj4f/50FUoPl6dsfTnkL/9LeY6TMfSL/mrs9GzAH77A2ieH234WAMs06qHsffvJ3YN0z4984dvE77HvLbF83lcaj5JmjvHsojxjeK9Whteox4oVx3ZuUMwh1zxxES59LtUpnlIeAcnJ25sqs86nZ6zWtSw67+ka01zFOZ4bFDPemTqmrmqemLoCf7iyd9K71EP7rgb/5X9J34Taky0APKWTRgD47bs15eTedxbBTm+oymaP4O5Ji2xd5yW+TxYTUQMA7wH4tRDioPqckGz89DuZQrwshCgWQhQXFhb6LWZkUTuGU5skqocJlNbujzZbYMqtWzQtnTW7VBuKG/r08W6IotKPpccm7Dtq7QzuiI3W3Y6D+j0ZLzDrjZjZ7rvBTgVlp3Wf7spiYeNetdtyPVbvPIzDx71zVueEZVsP2hoaivKopK+KgIhqQVICbwkhJsvBO4motXy+NQBvd1/PMNxUlVb+2dItmFEu0FZE3Y+NaYXiQna37h/s5JsbvW4lnp532+T7BeJdMsPFT6shAvBvACuFEE+rTn0EYJT8exSAD/2SgTHH1C2Dz1ZDcSas5LuJ123fy6/eCCCVNSv5rEzwher/MLAVc5p5GMT35uccwVkAfg5gKREpg173AxgP4G0iugnARgBX+ShDVmNo+iaXSO1Yv5vWctRb2gpRl9Ks5e5XFtvpLGhX0uo/J/2K2MpDbLo2+EFAZDPt0U2Cf4pACPENjL+7oX7Fy9RgVTZzNP1B9csyvzXq1Wl8MR0ZcpHv5k7frGsoe4rAiUTO7rValSvdz+UyXdjpXETZdbAM9727JMnk87GPVyCHCHuPHMfv3ltq+QwBgb/qOGObLu8zPHtt8laV6v1V//3NBvTt0ET3ufe+szjp+Fh5Ffo9NsNSHj06jrXniMwLlm45gD+4NPMEgN+oLEbUW26mw5x1pfjghxofNGYVotaixS4b9hzBpr3u9i620xsx2jbVigPHyi2Vkdrhmx52JvDHTrb+ZtKBkKrAb31DR14C1u6yb/Jbc5v/Co4VQUR5btbaFDPFLfuO4ZbXzT8INQTS3TfYLmpvlQqHyyqxWLMRu94G3XYJssfvhRLQ8uGi1DUSTlBMWxXMKsSlW52tb1Cw2gjdq6GhdKmoEoGUg3QVqR208n9q4LxO3YiIEuxrKKL4+eG5IcLDnBmBHxVihYVvAztR+jlZDGSGwzo7BGwxbRtWBBlMtjjzyiT8eGNeVLJ+GwOEWVa92FnN7hPsuplQwzuUZTER7RDoEidZsxGrqsdOReh3j8CLlnK65dALRWknD4VIb//sIGBFEFG8GBqKqjtlxhg/6gmrZ9obGvK5RxBiBek2Zru9pWoheGiIcYaTFtiBY/pL67VO5gCg9LD1zllmlFXYc7DFpMcel+9Hj10WFjUHjlq7ZvBbEZRVuN89aNfB9PJu3W793drssmXfUVtuOnYfOo7dBo4Ww4athiLC0fJk0zsnH56Rz571Op5DT318ps6V9jHz4slEE60jQS1q53JGaNeceM0wDxzUaS2w7HLBM+7irqgS+HZdqeV1dq4JC+4RRISjWmdnMRrViZGoTJr4PVlcauFUjvEXVgQRQfuZRdV8lMlOuDxmNqwIIoK2xeW3lQbDOIHLY2bDiiAiaL8ztvhhogT3CMKD1xFkEdqXzS0wJkqwHshsWBEEzDvzN+vu9nTl8zVb7C3ctA///mZDkGK5YoxPzryY6DB54Vbri5jYwuajAaM4nVLvk7r9wLEkU0+1UogDVtsIMgyTPkEME3OPIALY3dSaYRjGD1gRRIC47O7FMExmwoogArAaYBjGCLYaYhiGYXyHFUEE4JEhhmHChBVBAHy8ZBuKxkw19Py5ZEt6WxAyDJP58NBQhvDq7BIAwLrdqd5AAeBTeTN5hmGYMGBFEAF4+T7DMGHCiiAAuJ5nGCZdYr2gjIgmENEuIlqmCnuEiLYS0SL533C/4mcYhmHs4WeP4FUAw3TCnxFC9JH/TfMx/sgQ0f2qGYZhAPioCIQQXwHIyn0NF2zci106e5Nqh4h+3HEQG/YcwVK2GmIYxoAghpbDcDp3JxH9AsB8APcKIfbpXUREowGMBoAOHToEKJ57Rr4wBwUNamP+g+eZXjfs2a8DkohhGMaYoCeLXwDQGUAfANsBPGV0oRDiZSFEsRCiuLCwMCj5PGOPwZoBhmGYqBGoIhBC7BRCVAkhqgH8E8DpQcbPMAwTN4IwOgxUERBRa9XhFQCWGV3LMAzDBINvcwRENBHAYAAFRLQFwMMABhNRHwACQAmAW/2Kn2EYhrGHb4pACHGtTvC//YovyrD1KMMw6cK+hmLEqY/NQNGYqRCqRQM/++dczFq1K3E8ZfG2xO+iMVMDlY9hGMYI3rPYI0rlfXvLq6oTYd+uK8UPm/ajZ5tGAIDX52wMRTaGYRgzbPcIiGggEd0g/y4koo7+iZU5sJ8h99TO444rw/iJrS+MiB4G8DsAY+WgWgDe9EuoTIL1gHtYmTLZTXSczl0B4FIARwBACLENQEO/hGIYNUF4X2SYbMauIigX0iyoAAAiqu+fSJkFcXOWYZiIY1cRvE1ELwFoQkS3AJgJaWUwYwEBSZZEDMMwToiM0zkhxF+I6HwABwF0AzBOCDHDV8kYRqZlo9ooKT0athgMEwpN6tbyPQ5LRUBEuQBmCiGGAODKnwmcNk3qsiJgspbmDWr7Hofl0JAQogpANRE19l2aTISnCFwTl2mWWrkxEdRDsjHNmYjdBWWHASwlohmQLYcAQAjxK1+kyiD4M3EPWw0xjL/YVQST5X8MEzhx6RFICosNA5j4YXey+DUiygdwohy0SghR4Z9YmQObj7onNnkYEzEZRostRUBEgwG8Bsl1NAFoT0Sj5H2JGRVaS9EDxyqwcNP+cITJELh+ZRh/sTs09BSAC4QQqwCAiE4EMBHAqX4JxjAKOTHRBGGIWdS8HltUMa6xu6CslqIEAEAIsRqSvyFGA68d857YDA2Z8OEdZ3n+zBwCJo4+w/PnZgLDe7UKW4RYYbdHMJ+I/oUaR3PXAZjvj0jxRvBkoefEXw1kLjxBnhnYVQT/B+AOAIq56NcAnvdFopjDPQLvyYQs9atTw+WN8QK7iiAPwF+FEE8DidXG/i93iyH8XXpPVXU8cjUDRrCYLMXuHMHnAOqqjutCcjzHaGAHc95THZM8DWPhWzxyhok6dhVBHSHEYeVA/l3PH5HiQaexU3Hdv+amhD83a20I0mQ2XVo0CCyu5vXz0763sKFxJzkKq6M7FbL3eEYfu4rgCBH1Uw6IqBjAMX9EigfVApi9tjQl/JVvSoIXxoQzOzV3fM8ndw9CvfxcH6RxzsRbzsCDI3oGFt8nvx6kG96+WV3dcDWDuxW6ivuCni1d3W/FpFvibWH0n5v7o8BnB2wNa7vfxr1OrfS2Vn3r5v6YcudAPH9dP8wec65rOZxgV+JfA3iHiL4moq8BTAJwp39ixZeoWQ01S6OF26N1I7RtYl3xBcGZnZsj14OFBB0LjFvD6kq+RcM6utec3dW6kncrZd00lK+TocimLno7UWBAlwJ0aeFvr6ZhHfeKoF3T9AZLzupSgF7tGmN4r9aBf3+mioCITiOiVkKI7wF0B/BfABUApgPYEIB8sSMmw9lZh1klncOzvKHhVMf7bTeQCWtW0sGqR/ASgHL595kA7gfwDwD7ALxsdiMRTSCiXUS0TBXWjIhmENEa+W9TF7JHkkzRA5mSjgQm33ecFYGThocvqXT5UKd577cxhhdFIY4GI1aKIFcIsVf+fTWAl4UQ7wkhHgLQxeLeVwEM04SNAfC5EKIrJEukMQ7ljTxRKwTpDlVFLR1uMfu+7Xz7dnLDrDVpp4LJsCy3hdOK1+888kQRuH9E4FgqAiJSBs2GAvhCdc50ME12SLdXE3wZJOd1kP9eblPOUJmyeBt2HSzDtKXbceR4JaYs3pY49/nKnbjn7UWJ45iYvGcdZRXVhue4kg4Pp9ZUsTAljoGIWqxmRiYC+JKI9kCyEvoaAIioC4ADacTXUgixXf69A4ChmQQRjQYwGgA6dOiQRlTesGTLftw18YfEcZ/2TbBoc4030Ztei7anjQYeWEFkAlv3Gxu5RWVoKB3zTsWK5sazOmLCbPNpuyiOf9sV6c4h0gDEzYM64fa3Flpef81pHTBt6Q43onnKKe2bYPHm6HohNu0RCCGeAHAvpGGegaJmvCAHwF1uIpafZag7hRAvCyGKhRDFhYXuzPLccLisMul43e7DBleGz+izO6WE5eakZ8rmNcUnNMXvLzsp0DgHdimwdZ2b+vF3w7pbXnNGp2a64eecWJi01WPjNDYpr5ufi5LxIzDuEnMT21FnnpB03KSedVwl40fgp6e2Sxx3KqyPa05r71jGkvEjDM9Z5f3gboUoGT8C913YDQAwvFdr1M6rKdO/kcPVPDiiB84+0bjOKBk/Aq/ccJqF1OmjV6ndMbizb/F5gWVzUQiRsmpK9j6aDjuJqLUQYjsRtQawK83nBIb2pUavTWVOdZpjVV73biPYGE1gr0fgLkeMolCPdPg56iGQXtlVi+RHz8nqmVYxRrFcxXF+Lejm4kcARsm/RwH4MOD4HaN9p3F7xVEZUyWiwMfZ7U6UuxkyUccRxUpJwYu89yN51hW9P5lq9FS/ymg0vkJjfFMERDQRwBwA3YhoCxHdBGA8gPOJaA2A8+TjSBO1BWJOqYqIIojy5jK2+gNZWkGo8aNHYFXR6531Qgw/50vi9E4VfJtJFEJca3BqqF9x+oG2AohwfaZLukNDXpdmAgXeZbYbnZ1pFKNnuU2SOk/8zB0B4boC1bvf7fdgJVOUe1lGRKTt5Qg2KTHgqc9WYc3Ow2jVONnlwEHN5HHUqZ+m1ZCdyUQn5OSk50LBDY3q2EuDm5au2ieTkYVWFBzOqdsDzernpzVk6MfwnnXep563m5/5eTkor9Q3GzZ6QtN6+diyz50bNb1RhPzcaBhtGBFt6ULk71+sxfTlO/DqtyVhi6LLHUP0rRCev65f0vHY4T1SrunfUd+KRc2L15+KTib+eZxCIIzs1w6/HdYN9VWV5wvX9cN5Pdw7W+veqmHScX5uDsaP7GXrXrUVipZLT2mDf/6i2PD89WfUWOPcMcR4jaVSeXVrmSynXr06qGuBLSd3TiEi/Glkb3xwu/fbZqaLmyHDj+8aaHp++t2D8KeRvXXP6emfJ6/ohX+NKsaV/dqmLdMLmu+voEE+xl7UHecYWDE9fvnJacflJawIYsqwk1rrhg/vlRyu10r9xZlFls9v0agORg2wvs4uREBebg5uH9wFdWrVKILzerZEr7aNXT+/3wnJ3kr+dm1fNKnn3snaKe2b4PyeLQ3nimqpWnrqdBmhrYCSrYakgy4tGiBPZ7zKTatSieeq09qjQ3OcYMVhAAAc8ElEQVT7TtH8GOZQu/q2Mm/WHY6SwzoW1DftHXQqbICrbJq7vvd/A/Cz/h3QslGdJOXulIt6tU7Kswa183DrOZ2RY6Dx3MTlJawIGEO8HJ81GgLwKgo3z7Ez1OC2QnSSl/4MJUVn4FqdF+p1FHbxxgLKOF738z7u7g8DVgRMIKg//jhNAMZIVFO8qpy8saKryVUrF+N+5b95GYxhTe4SVgSMIV5+hIY9gihoBZ9FUCcxrNZilFqp6vzIs1IEJkNDxvc4f6HqW7z0FxahbDeFFUFMCaT+9DASo+/dbgx+mhn6v67YmWM7w1XILqSI0noYdfKsewQmQzhI/72b3eapSXB0st0UVgQZxGlF1tZAANCtlb09gE9u08iNOEkM7tZCdeT86/1Jv3bWF/lAr3buJ7LP7d4i4RzuwpNbJcKHdCvE+fL2lC0b1WzBaJQ7l/Ruk3R8pYM80VZIl53SRv9ChzitiM/t3kLTI3A+WXxFX8mqJ535hZoHG59q29SdxZY6q7XGG1GFFUGMWfn7YYl9cp+8oleiUjFj8bgL0KVFQyx86HwsfvgC02v7dqixxFn26IV446bTHcn3hysl880h3QpxXX99D7J2K5I/XNkLSx4xltfpBOuL19eY+RnJ8MND5yeUq9OW3ZQ7a0wbbxrYEQUNamPxuAvw66FdE+GjBhQlvGpqN75Xi6Q4oxszvDtevP5UAMBJbRph/JX2zGP1GHdJsgNAo2e57UlcolE4L/38VNN3teL3F6Jn65oGiN67+f1lJ2PJIxegdp5361LU0RhtE3lmp+aY6GDf5yl3DsRvdZziFZ8Qvf24eEFZjKmbn4s68sfQ1OYCsMbydU73Mm5QOw8NbS7QUlDs85vUy9eM2zqvXPJyc9DIw0U5alNPo4rJzaI65d4m9Wol0t5Y8zwiSpo7sap0c4nQqK70yTaonYc8B/mhfbJ2SMbuwkOnCrG+ZhFhrdwcU8OBevl5qG2x+XtuDtleLGhEOpZZDevkoaCB9Xej5FHjurV0zUbzTdathEX0JGIcoVQeQcwZpBuFmWuJsCaLgxq6TSd1fmRJumPVfpiyOnmiX6uytXlstxwarQdQE+Q36RWsCGJOzQfuf6mLcsF2LJuqYjS6lxy01l3LY+uZLrykpqkJ3A4NubbwiVCZI5J6ZV48J2qwIog5QRolRMFnjl0cLeDywWwonUrb2LFdTExPbKLOG9dO6wK+zwsPrFF8nawIYo6VyaGXOI1DuT6Icu+8Q+CvVOm8DkUiIyVCSF8ZW6U2fTNMdzb7fnQI/LpHCGeeaqPY8jeCFUFEuLxPG92CozYrVKM4WXvk0p4Y0bu1rlOr/9zSHwBwq7yFpduC2aWFPbNTBbuVxA0Di2w/U2+7y1d+ab3t4Lu3nQkAOLGllAZ1q0wr57u3nYmfu/QBYzeve7RuiCv7tsWzV/dRyWNMugpMr1fx1s39Ewu6jBz/abfivF/jxFApY2pevL4fWjWqkxKuYJU3f/npKYbxabm2fwdcdHIrjFSZ0v60WN+s9tFLT8IfZUeE/U5oisv6tMG/flGMK/q2xUk2TaW1PYJ7zz8x5RpLhR5BBcGKICJc3LsN3rix5qNSHLGdqPFW+fx1/VAyfkTCYqRd03r4x8/66To8G9BZ2rNXb1/XdLDjVC0dGtWphZLxI3C6jXUQeg7zhnRvkXqhhuKiZigZPwLtmqY6XNN+mMVFzfCYxiukXvWrKBU97CrBvNwcPH11H3Rpkfye9W53U4HoyX9WlwKsfXI4SsaPSHm3I3pL9u8tNRV60/r5eP1GyYx4YJeCJBNjhWEnt8ZdQxVPrKlCq/NGL02dC2vytY2BKadCozq18ML1p6JZ/RorIiPrtlEDinD1aZIZc63cHPz1mr44r2dLPHN1H1sWWETJiuCUdo1x19CuhmbVEazvDWFFEBH8bCWE7cbB7piom+Eav9OY7ji9E7nsRJH20JAPI2FpDyeph4ZiVV1qVsgbZAAPDTFp48c2gGET5SR5UTGaPcNV2g3dcbiwGkr7Tu/xo1gEMQErhEaxG0Yqm4/GSMmxIogKKXbN0t8oWhjEGSWbvchWs2dErQrwwvJImyazR5orSfOhoShjZyMd7hFkAOWV1ajy0v2gTQj+F5ywyqU2N8NUbjUKVqjC7CwSchqR0xuMh8b0QsPIQq/ijFH9mIR2jsDqg41TOlkRaDjxwU/Q+f5pgcebY7AfbFGB/d2kFFpr9llWCmTxCfac0mlRfN34jRsFYfRNNk9xpZF8YcPaeejRWjNRq4N20t5SHhfVAPkwqOCH4ki74UK6P10RlGJUK4LOhfpbuVqt8YzikBH7GooIRh/VT05tj0t6t8Ers0swffkOW8+a9qtB2H34eOI4J4fw8V0DcYLJFoWv/PI03PDq96ij8fMy856z0dRky8euLRpgza7DieMJvyzGja/OtyVnEMy45xyUqvJCQUBKW5N6+WhStxZe+nK96XNuO6czVu04hI8Wb6t5ho7mmjt2KMoqqtKqJJ0oQqvH3z+8O56c9qPq4c7lcRqn7j161k9WFwD4+rdDLF1UB40QAKk+jxvP6pgIT77OfI4gikNG3COICNpCoz7q36m5o8LTtH5+Sgv25LaNTZ3GtW8mKYk2jZPN9bq0aIjmDfTXMgBA3w5Nko5PapPqtjmIlbFGH12z+vnoqsoLdT52adEQBQ1q2zIdzM0hFBdZe41s1bgOigr0W4p2sbV3gcX5wd1aJPXkqj14B0LzN13srCxu36yepeloGOQ4mN+IYoVvBPcIIkIOhWvZ4VWhTVqoFeEvIah5itDmZSJsZGDDAjOy2Joslv/GKWmhKAIiKgFwCEAVgEohRHEYckQKAkS19WV+feBR8E4ZrCsK71rIgeEgq7UTz95s+G5fDFOLqjjVkCpSJosNqLEaik9Cw+wRDBFC7Akx/kghVaipn0/gDsd8KLuBVPA25fb02/QpYZIFmbvxZSE0WyZGaCVB0spij54Z3DoCO9cpcwTxIavnCN5dsEV3IjEMckhTyMJqTXj4QTlNgRul5zyutKPyDS+Vvh/p826OQP07TtVlMla94TglLSxFIAB8RkQLiGi03gVENJqI5hPR/N27d3suwJZ9R3HfO4vxf28u9PzZZlzeR3+v2O6tGyV9vHfLvloUR29+Faor+7bFkG6Ftp//4IhkJ2BEQIuGtdG7XWPk5+WgqcrnS/9OkrnqDQOKku7xow4eeaq9/XuVjzcdGYZ0S/ZpdJ9HPpy0mL2LHq0k52i3D+6ccm7cxT0BSBPbJzSvl7QdpVPFoH1n6aIk5ZcDitBPNiy4R8dRW1yopXI/arXuQ6sorjmtPQDoOojU44Tm9XDfBcHkVVhDQwOFEFuJqAWAGUT0oxDiK/UFQoiXAbwMAMXFxZ7XHeWV0oD87oB7BA9d3BPPXtMXRWOmJsI6FdZPsvA4q0tznNu9JUrGj0iE+dWCfVr2erlWZQJqxs2DOuHmQZ3wn+824f73lwIA5j1wnu61LRrWSUqDn/Ro3Qgl40fg5tfmY+bKnYZtNTcKtX2zeigZPyLx7sw2Jk/nfVl4LAAgbXdplKc3DuyIGwd2TBz/5NR2aFgnD7e+sQBO1kgaPd/JHIGWRy6tUUoXnNQq5ZluUSplbUMlXZ644mQ88P6ylPCcHEKP1o2wcvtBM2EkNIkbP7I3xo/sjU+Wbrclw5e/GWJTWveE0iMQQmyV/+4C8D4AZ7uieyFD0BHK6HWFnSzdZ+JFOorHywl4PybHvSyeXvd0vRpqMnefYS8HjESJ4ucduCIgovpE1FD5DeACAKmq12cSM/tJYUHYu5ucMznpu/uJCIxnBvmBZIuyjfoYfBRX2boljkUrjKGhlgDelwtoHoD/CCGmBy9GqiYIonLQXWkpB8axcop4PZNCza5p0ctsPyXy0nzU0wh9WL/iBeaNMnOh42g1FLgiEEKsB3CK5YW+yyH9Vb+sqhBXwIaN06RHsSK1Q2KyOMLie6lccxKKzzuiWYIlwjBHTXGqqMgSo1ZS1pmPFo2ZiqIxU1Ne1rSl29H1gU98jz8313iOwKzcNJOdp9XL92eXMKc0qC21IZqY+CEKA8XJXF2DfGreIJh8VLaA1O7wZRf1FqWtZCeC6fjeqZcvvSczf1FO8VKp1M7L3CooPmogi11MaHsET05bGUi8SgX65BW9ElY3dhoODwzviR6tG2FwN3umZ05x2ni5pHcbHCyrxE91zDZn3nM2Nu89Zniv0VyMNnjy7QOcCQVg3CU90bt9YwzsUqB7fuxFPXBiy4Y418b2lm5oWj8fz17dBwO6NLd9jzr9z13bD30fmwEAePPm/pi7vjRRdpxwRqdmePKKXrjUwGzZb6zKVWHD2njm6lPw//67OBiBbOJE2WWCsUfmqmMLlKGNIDeAaVgnT/d3QiYTGerm5+K6/idEpruZk0P4+Rkn6O5j3KVFQ1v7CFvRT2c/XCvq184zzae6+bm4/oxg8vHyvm3RoqH9HkGiTEJSJAotG9XBZX3apiUDEeFn/TukpUSCgEC4oq+9NSC2nhfC55E6NJRct8SBrFUECsqYsRfeGa1QR6He/Caq8wZ+YVQJx7Ah5Q8RrUGy6f148Qbi9F1nrSLQ1vtB7EqmHhKp1Ikvot+/5wTuP4kJnLi/YTfyx7F4Z70iUCrfIHoE6rq/Wt0jyBIFwJgT9QoknWIaVKs4So2LGkOUUMVwRFYoAiEEJi/cgs17jybCvl1X4/h00rxN2HO43Hc5qi16BEx2E0c/9m7xfGWxt49Ljxh+2lmhCD5YtBX3vL0Yg/40KxH2+FTJSujHHYcwZvLSQORQl4/e7Wp28rqib3oTgV5SIO9CdmW/EGVRKcrTi8z3V+7Tvonp+ahzSgzlv7yvZHnUSd6BTfG1lJdDaNc0eTexvu2liX4rK7ehPltvpUtfnfdzocpHElBTXLW9kevO6AAAqGWw85228zKgs33LMr+IpimBx2wqNTZlDBJ1gTm5bY0iGH12pzDESaJx3VpY/fhFqKWzziFo3r3tTPQ1sRha+8RFICJ0vn9agFJ5hyJ/3LhlkFRO2zerh1WPD0O+XNH9+NiwlPfRq11jrHp8GGrnma/XuMjEcZ8TvG6En9xWkj8vJwdCCFQJYZkWxWR53MU9MfaiHrbXfbx5U/9AFrOakRWKICoYjQZFpVLIj8jinrzcHNOPyM4ew1HGUH7FNUE0ikMK6nKqrhSN0mNVcfqBl99SjfykW1EqUSlxqo/z8+zLkZNDyAl5UCveX5RNovJhRWlCi2EYdxgNDcWR7FAEYQsgw/PDEjbc7jOMIzKgLg6VrFAETLyIiuIOGqOdrRj7BNn71w4NxZmsUARxeU+KmDlxEThN8gzG/zM93VYoFUoavuWYEEh3aCiKxTwrJov90NgTbzkD1/5zruV1nQrq4+7zuuLuSYssr+3fqTl+OaAIt54TvhWRnzx9VR/8YsI8FDWvlxT+92v74pXZJeilsqiKIg9f0hMntmyYOH7jptOxfvcR188dfXYn7Dl8PLHd5IvXn4pDZRWunxsUv7mwW2imkGZu0b3OR+NtUO3VM+f3bAkAePqq0L3xJ8gKReA1rRrVwZkGBf7yPm3wwaJtAGr2fq2sqralCHJzKGlv10ylTRPJ5jxH0/Rt36wexl3SMwyRHHHDWR2Tjgd1LcSgru69wjaonYcnr+iVOB52ciuTq6PHHUO6hC2CLl7no9vpiFq5OYHt5W2XrBgaCptsH/IwhCf4mBiTSZ81K4I0cFoAMqnAMEyUCfJTyyRLpaxQBEH6M9EbJ8wEqwJf4GxhYkjKRjShSOEtGT1HIITA1v3HsGjTfk+fyxW7WzLh02GiRJCt80wsvRndI3jpq/UY+MdZ+GzFTt/i0LpCOKlNoySHcmpOK0r2n9O2SV3d6xh7NKsfrf2SmfRp6NUOaiE00jKhWZjRPYJPlu3w9Hm3nt0JL321PlHWvh1zLurWyk3sLUsE3DSwI645vQNKDx9Puvd/9w1GYcPaSWHT7h6Eg8fiYx7oHd58OrPuHYxDx7Mx//xhwYPnoaIqtb07d+xQ350Rfv27IThSXuVrHF7BQ0Mxo6Ky2tPnndG5OV76an3iuI2mRd+jVSMQERrUzkvZI7ZIdt2rpnHdWmhct5anMsYDbz6dxvVqoXG9bMw/f2jeoLZueKvG9vddTpcm9fLRpJ71dUZEoTKOc88glKEhIhpGRKuIaC0RjfErnooqbxWBiLh3SIbJdsL8NKOgjNIlcEVARLkA/gHgIgA9AVxLRL6sIvJaEVTLj2NfMAzDZBJh9AhOB7BWCLFeCFEOYBKAy/yISG+80w1x3Is0ynA2Ml4RBZv+OJfnMBRBWwCbVcdb5DDPKavwdvJJ2XM4zi+cYTIZbqSlR2Qni4loNIDRANChQ4e0nlF6xNsN6ds3lWazjNYR3D+8h6fxZSrtmtZDu6Z18dDF0fErdP0ZHXi/iAC5sm9bFDTUn5xOhxvOKsKny3fg/B4tPXumEQ+M6IHfvrsEnQsbAAD6dWiKwoa1cfd5XX2P2y8o6N11iOhMAI8IIS6Uj8cCgBDiD0b3FBcXi/nz5zuO69Y35uPT5fbXEHzzuyFoJ1f2j05ZjldmlwCocR63bvdhDH3qS3QsqI9Z9w1O3Fc0ZmrSdQzDMFGAiBYIIYqtrgtjaOh7AF2JqCMR5QO4BsBHIciRgnpxmN6EsKIzuffJMEwmEfjQkBCikojuBPApgFwAE4QQy/2Iy2lXX9050h/9YU3AMEzmEcocgRBiGoBpAcTj6Ppq1fWmjuXSlIdhGCaKZLSvIa97BImhITZNYBgmg8hoRdCuqbFTt7q1clPC8vPMs6OOfE97k+cyDMPEjciaj3rBuIt74tt1pXhgeA9UVgvsPnQc97+/FIDk5Kr48ZkAgDaN6+CWszuhZaManypKq39Q14JEWPtm9fDi9f1wZucCqPnqN0Ow7cAxv5PDMAzjCxmtCPJyczDznnMSx4fKKnD/+0tRLz8XBSoHWyN6t07Zh1YZ/BmgqfSHndw6JZ4OzeuhQ3MXHrMYhmFCJKOHhrQoewdr55B1x/zlIBFrV1IMwzDWZJUiIIPKXV8P8IQwwzDZQXYpAoPKPUd3n2HpbxScWTEMw/hJVikChZShIZ1rlLCgXXAwDMMETVYpAsU89NazOyWF6/UIhvZoAQAY2LXQf8EYhmFCJKOthrTk5lCSY7h7zj8RT89YjRydLsGpJzRjJ3IMw2QFWdUj0FJds1Q4XEEYhmFCJKsVgaIH9HoEDMMw2UKWKwJlxzHWBAzDZC9ZrQiquUfAMAyT3YogL1fSAFbO5hiGYTKZrLIa0jL67E44Vl6FUQOKwhaFYRgmNLJaEdTLz8NY3nCeYZgsh8dEGIZhshxWBAzDMFkOKwKGYZgshxUBwzBMlsOKgGEYJsthRcAwDJPlsCJgGIbJclgRMAzDZDkUhx24iGg3gI1p3l4AYI+H4oQJpyV6ZEo6AE5LVHGTlhOEEJa7a8VCEbiBiOYLIYrDlsMLOC3RI1PSAXBaokoQaeGhIYZhmCyHFQHDMEyWkw2K4OWwBfAQTkv0yJR0AJyWqOJ7WjJ+joBhGIYxJxt6BAzDMIwJGa0IiGgYEa0iorVENCZseawgohIiWkpEi4hovhzWjIhmENEa+W9TOZyI6G9y2pYQUb+QZZ9ARLuIaJkqzLHsRDRKvn4NEY2KUFoeIaKt8rtZRETDVefGymlZRUQXqsJDLX9E1J6IZhHRCiJaTkR3y+Gxey8maYnje6lDRPOIaLGclkfl8I5E9J0s13+JKF8Ory0fr5XPF1ml0TFCiIz8ByAXwDoAnQDkA1gMoGfYclnIXAKgQBP2JwBj5N9jAPxR/j0cwCcACMAZAL4LWfazAfQDsCxd2QE0A7Be/ttU/t00Iml5BMB9Otf2lMtWbQAd5TKXG4XyB6A1gH7y74YAVsvyxu69mKQlju+FADSQf9cC8J2c328DuEYOfxHA/8m/bwfwovz7GgD/NUtjOjJlco/gdABrhRDrhRDlACYBuCxkmdLhMgCvyb9fA3C5Kvx1ITEXQBMiah2GgAAghPgKwF5NsFPZLwQwQwixVwixD8AMAMP8lz4Zg7QYcRmASUKI40KIDQDWQip7oZc/IcR2IcRC+fchACsBtEUM34tJWoyI8nsRQojD8mEt+Z8AcC6Ad+Vw7XtR3te7AIYSEcE4jY7JZEXQFsBm1fEWmBecKCAAfEZEC4hotBzWUgixXf69A0BL+Xcc0udU9qin6U55yGSCMpyCmKRFHk7oC6n1Gev3okkLEMP3QkS5RLQIwC5IinUdgP1CiEoduRIyy+cPAGgOD9OSyYogjgwUQvQDcBGAO4jobPVJIfUHY2nmFWfZZV4A0BlAHwDbATwVrjj2IaIGAN4D8GshxEH1ubi9F520xPK9CCGqhBB9ALSD1IrvHqY8mawItgJorzpuJ4dFFiHEVvnvLgDvQyogO5UhH/nvLvnyOKTPqeyRTZMQYqf88VYD+CdquuCRTgsR1YJUcb4lhJgsB8fyveilJa7vRUEIsR/ALABnQhqKy9ORKyGzfL4xgFJ4mJZMVgTfA+gqz8TnQ5pk+ShkmQwhovpE1FD5DeACAMsgyaxYaYwC8KH8+yMAv5AtPc4AcEDV3Y8KTmX/FMAFRNRU7uJfIIeFjmb+5QpI7waQ0nKNbNnREUBXAPMQgfInjyP/G8BKIcTTqlOxey9GaYnpeykkoiby77oAzoc05zELwE/ky7TvRXlfPwHwhdyTM0qjc4KcLQ/6HyQriNWQxt8eCFseC1k7QbIAWAxguSIvpLHAzwGsATATQDNRY3nwDzltSwEUhyz/REhd8wpIY5U3pSM7gBshTXqtBXBDhNLyhizrEvkDbK26/gE5LasAXBSV8gdgIKRhnyUAFsn/hsfxvZikJY7vpTeAH2SZlwEYJ4d3glSRrwXwDoDacngd+XitfL6TVRqd/uOVxQzDMFlOJg8NMQzDMDZgRcAwDJPlsCJgGIbJclgRMAzDZDmsCBiGYbIcVgRMRkNEVSrPlIusvE0S0W1E9AsP4i0hooI07ruQiB4lyUPoJ27lYBg75FlfwjCx5piQlvLbQgjxop/C2GAQpIVFgwB8E7IsTJbAPQImK5Fb7H8iaf+HeUTURQ5/hIjuk3//iiT/90uIaJIc1oyIPpDD5hJRbzm8ORF9JvuX/xekxVlKXNfLcSwiopeIKFdHnqtlJ2S/AvAsJHcJNxBRZFfDM5kDKwIm06mrGRq6WnXugBCiF4DnIFW+WsYA6CuE6A3gNjnsUQA/yGH3A3hdDn8YwDdCiJMg+YnqAABE1APA1QDOknsmVQCu00YkhPgvJI+ay2SZlspxX+om8QxjBx4aYjIds6Ghiaq/z+icXwLgLSL6AMAHcthAACMBQAjxhdwTaARpM5sr5fCpRLRPvn4ogFMBfC+5y0Fd1Dh503IipE1fAKC+kPzuM4zvsCJgshlh8FthBKQK/hIADxBRrzTiIACvCSHGml4kbU1aACCPiFYAaC0PFd0lhPg6jXgZxjY8NMRkM1er/s5RnyCiHADthRCzAPwOkuvfBgC+hjy0Q0SDAewRkl/8rwD8TA6/CNKWjoDk3O0nRNRCPteMiE7QCiKEKAYwFdKuU3+C5AytDysBJgi4R8BkOnXllrXCdCGEYkLalIiWADgO4FrNfbkA3iSixpBa9X8TQuwnokcATJDvO4oa98CPAphIRMsBfAtgEwAIIVYQ0YOQdp7LgeTR9A4AG3Vk7Qdpsvh2AE/rnGcYX2Dvo0xWQkQlkNws7wlbFoYJGx4aYhiGyXK4R8AwDJPlcI+AYRgmy2FFwDAMk+WwImAYhslyWBEwDMNkOawIGIZhshxWBAzDMFnO/wcE5gfNqq1IkwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def dqn(n_episodes=2000, eps_start=1.0, eps_end=0.01, eps_decay=0.995):\n",
    "    \"\"\"Train the Deep Q network.\"\"\"\n",
    "    highest_score = 0\n",
    "    scores = []\n",
    "    scores_windows = deque(maxlen=100)\n",
    "    eps = eps_start\n",
    "    solved = False\n",
    "    for i_episode in range(1, n_episodes + 1):\n",
    "        env_info = env.reset(train_mode=True)[brain_name]\n",
    "        state = env_info.vector_observations[0]\n",
    "        score = 0\n",
    "        while True:\n",
    "            action = agent.act(state, eps)\n",
    "            # send the action to the environment\n",
    "            env_info = env.step(action)[brain_name]\n",
    "            # get the next state\n",
    "            next_state = env_info.vector_observations[0]\n",
    "            # get the reward\n",
    "            reward = env_info.rewards[0]\n",
    "            # see if episode has finished\n",
    "            done = env_info.local_done[0]\n",
    "            agent.step(state, action, reward, next_state, done)\n",
    "            # update the score\n",
    "            score += reward\n",
    "            # roll over the state to next time step\n",
    "            state = next_state\n",
    "            \n",
    "            if done:\n",
    "                break\n",
    "        scores_windows.append(score)\n",
    "        scores.append(score)\n",
    "        eps = max(eps_end, eps * eps_decay)\n",
    "        if np.mean(scores_windows) > highest_score:\n",
    "            highest_score = np.mean(scores_windows)\n",
    "        print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_windows)), end=\"\")\n",
    "        if i_episode % 100 == 0:\n",
    "            print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_windows)))\n",
    "        if np.mean(scores_windows)>=13.0 and not solved:\n",
    "            print('\\nEnvironment solved in {:d} episodes!\\tAverage Score: {:.2f}'.format(i_episode-100, np.mean(scores_windows)))\n",
    "            solved = True\n",
    "        if np.mean(scores_windows)>=13.0 and solved:\n",
    "            if np.mean(scores_windows) == highest_score:\n",
    "                print('\\tModel Saved')\n",
    "                torch.save(agent.qnetwork_local.state_dict(), 'checkpoint.pth')\n",
    "    return scores\n",
    "\n",
    "scores = dqn()\n",
    "\n",
    "# plot the scores\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(len(scores[:2000])), scores[:2000])\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and observe the trained agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score: 21.0\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "checkpoint = torch.load('checkpoint.pth')\n",
    "agent = DQNAgent(state_size=37, action_size=4, seed=0, use_dueling=True, use_double=True)\n",
    "agent.qnetwork_local.load_state_dict(checkpoint)\n",
    "\n",
    "\n",
    "env_info = env.reset(train_mode=True)[brain_name]\n",
    "state = env_info.vector_observations[0]\n",
    "score = 0\n",
    "while True:\n",
    "    time.sleep(0.025)\n",
    "    action = agent.act(state, 0)\n",
    "    env_info = env.step(action)[brain_name]\n",
    "    next_state = env_info.vector_observations[0]\n",
    "    reward = env_info.rewards[0]\n",
    "    done = env_info.local_done[0]\n",
    "    score += reward\n",
    "    state = next_state\n",
    "    if done:\n",
    "        break\n",
    "    \n",
    "print(\"Score: {}\".format(score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch0.4",
   "language": "python",
   "name": "torch0.4"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}