multi-arm-bandit-problem.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reinforcment learning "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's define a few bandits "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "bandits = np.random.rand(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.05816226  0.36539606  0.50160369  0.45681616]\n"
     ]
    }
   ],
   "source": [
    "print(bandits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# so we have four bandits, of course you can easily implement a different number of bandits but for now \n",
    "# let's have four "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def banditaction(bandit):\n",
    "    randomnum = np.random.randn(1)\n",
    "    if randomnum >= bandit:\n",
    "        return 1\n",
    "    else:\n",
    "        return -1\n",
    "# We want the agent to learn to always choose the bandit that gives the positive reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# so let's start:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = tf.Variable(tf.ones([len(bandits)]))\n",
    "chosen_action = tf.argmax(weights,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# placeholders\n",
    "reward_holder = tf.placeholder(shape=[1],dtype=tf.float32)\n",
    "action_holder = tf.placeholder(shape=[1],dtype=tf.int32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight = tf.slice(weights,action_holder,[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# As a loss function I take -log(policy) * advantage. policy = chosen action's weights, \n",
    "#                                                     advantage = how much better this action\n",
    "#                                                     was than baseline.\n",
    "loss = -(tf.log(weight)*reward_holder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's use GradientDescentOptimizer with some learning rate (this is a hyperparameter so it can be tuned)\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# and now update, we of course want to minimize loss\n",
    "update = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some basic settings "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 10000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_reward = np.zeros(len(bandits))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "epsilon = 0.1 # chance to take a random action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "initialize = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ok we are ready to launch the session, at least I think so"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(initialize)\n",
    "    i = 0\n",
    "    while i < epochs:\n",
    "        #Choose a random action or one from our algorithm\n",
    "        if np.random.rand(1) < epsilon:\n",
    "            action = np.random.randint(len(bandits))\n",
    "        else:\n",
    "            action = sess.run(chosen_action)\n",
    "        # now we are ready for a reward\n",
    "        reward = banditaction(bandits[action])\n",
    "        # and now we need to update\n",
    "        up, respo, wei =\\\n",
    "          sess.run([update, weight, weights], feed_dict={reward_holder:[reward],action_holder:[action]})\n",
    "        #\n",
    "        # add to total_reward\n",
    "        total_reward[action] += reward\n",
    "        i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The algorithm thinks that 1 is the best to play\n"
     ]
    }
   ],
   "source": [
    "print(\"The algorithm thinks that \" + str(np.argmax(wei)+1) + \" is the best to play\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Yes, that's correct\n"
     ]
    }
   ],
   "source": [
    "if np.argmax(wei) == np.argmax(-np.array(bandits)):\n",
    "    print(\"Yes, that's correct\")\n",
    "else:\n",
    "    print(\"nah, it's wrong\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}