Exp12.ipynb

{
 "metadata": {
  "name": "Exp12"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%load_ext autoreload\n",
      "%autoreload 2"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#SCIRL on the Highway\n",
      "from DP import *\n",
      "from stuff import *\n",
      "from pylab import *\n",
      "from random import *\n",
      "import numpy\n",
      "from rl import *\n",
      "P = genfromtxt(\"Highway_P.mat\")\n",
      "R = genfromtxt(\"Highway_R.mat\")\n",
      "Gamma = 0.9\n",
      "ACTION_SPACE = range(0,5)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "Highway = MDP(P,R)\n",
      "mPi_E, V_E, Pi_E = Highway.optimal_policy()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "MDP OK3\n",
        "Shape of P (3645, 729)\n",
        "Shape of R (3645,)\n",
        "Card of S 729\n",
        "Card of A 5\n",
        "Card of SA 3645\n",
        "Iteration 0, 888\tactions changed."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Iteration 1, 800\tactions changed."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Iteration 2, 234\tactions changed."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Iteration 3, 50\tactions changed."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Iteration 4, 0\tactions changed."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rho = lambda : int(rand()*729) #uniform distribtion over S\n",
      "l_D_E = [array(Highway.D_func(Pi_E, 1, 7,  rho)) for i in range(0,7)]\n",
      "D_E = vstack(l_D_E)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#SCIRL\n",
      "#Precomputing mu with MC and heuristics\n",
      "def MC_mu(episode):\n",
      "    answer = zeros((729,1))\n",
      "    for i in range(0,len(episode)):\n",
      "        answer[episode[i]] += pow(Gamma,i)\n",
      "    return answer\n",
      "\n",
      "feature_expectations_MC = {}\n",
      "d_mu_MC = {}\n",
      "for episode in l_D_E:\n",
      "    for start_index in range(0,len(episode)):\n",
      "        data_MC=episode[start_index:,:2]\n",
      "        state_action = data_MC[0,:2]\n",
      "        state = data_MC[0,0]\n",
      "        action = data_MC[0,1]\n",
      "        mu = MC_mu(data_MC[:,0])\n",
      "        try:\n",
      "            d_mu_MC[str(state_action)].append(mu)\n",
      "        except KeyError:\n",
      "            d_mu_MC[str(state_action)] = [mu]\n",
      "        for other_action in [a for a in ACTION_SPACE if a != action]:\n",
      "            state_action=hstack([state,other_action])\n",
      "            try:\n",
      "                d_mu_MC[str(state_action)].append(Gamma*mu)\n",
      "            except KeyError:\n",
      "                d_mu_MC[str(state_action)]=[Gamma*mu]\n",
      "for sa in d_mu_MC.keys():\n",
      "    feature_expectations_MC[sa] = mean(d_mu_MC[sa],axis=0)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 21
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Structured Classifier\n",
      "class GradientDescent(object):\n",
      "    \n",
      "   def alpha( self, t ):\n",
      "      raise NotImplementedError, \"Cannot call abstract method\"\n",
      "\n",
      "   theta_0=None\n",
      "   Threshold=None\n",
      "   T = -1\n",
      "   sign = None\n",
      "        \n",
      "   def run( self, f_grad, f_proj=None, b_norm=False ): #grad is a function of theta\n",
      "      theta = self.theta_0.copy()\n",
      "      best_theta = theta.copy()\n",
      "      best_norm = float(\"inf\")\n",
      "      best_iter = 0\n",
      "      t=0\n",
      "      while True:#Do...while loop\n",
      "         t+=1\n",
      "         DeltaTheta = f_grad( theta )\n",
      "         current_norm = norm( DeltaTheta )\n",
      "         if b_norm and  current_norm > 0.:\n",
      "             DeltaTheta /= norm( DeltaTheta )\n",
      "         theta = theta + self.sign * self.alpha( t )*DeltaTheta\n",
      "         if f_proj:\n",
      "             theta = f_proj( theta )\n",
      "         print \"Norme du gradient : \"+str(current_norm)+\", pas : \"+str(self.alpha(t))+\", iteration : \"+str(t)\n",
      "\n",
      "         if current_norm < best_norm:\n",
      "             best_norm = current_norm\n",
      "             best_theta = theta.copy()\n",
      "             best_iter = t\n",
      "         if current_norm < self.Threshold or (self.T != -1 and t >= self.T):\n",
      "             break\n",
      "\n",
      "      print \"Gradient de norme : \"+str(best_norm)+\", a l'iteration : \"+str(best_iter)\n",
      "      return best_theta\n",
      "\n",
      "\n",
      "class StructuredClassifier(GradientDescent):\n",
      "    sign=-1.\n",
      "    Threshold=0.01 #Sensible default\n",
      "    T=40 #Sensible default\n",
      "    phi=None\n",
      "    phi_xy=None\n",
      "    inputs=None\n",
      "    labels=None\n",
      "    label_set=None\n",
      "    dic_data={}\n",
      "    x_dim=None\n",
      "    \n",
      "    def alpha(self, t):\n",
      "        return 3./(t+1)#Sensible default\n",
      "    \n",
      "    def __init__(self, data, x_dim, phi, phi_dim, Y):\n",
      "        self.x_dim=x_dim\n",
      "        self.inputs = data[:,:-1]\n",
      "        shape = list(data.shape)\n",
      "        shape[-1] = 1\n",
      "        self.labels = data[:,-1].reshape(shape)\n",
      "        self.phi=phi\n",
      "        self.label_set = Y\n",
      "        self.theta_0 = zeros((phi_dim,1))\n",
      "        self.phi_xy = self.phi(data)\n",
      "        for x,y in zip(self.inputs,self.labels):\n",
      "            self.dic_data[str(x)] = y\n",
      "        print self.inputs.shape\n",
      "    \n",
      "    def structure(self, xy):\n",
      "        return 0. if xy[-1] == self.dic_data[str(xy[:-1])] else 1.\n",
      "        \n",
      "    def structured_decision(self, theta):\n",
      "        def decision( x ):\n",
      "            score = lambda xy: dot(theta.transpose(),self.phi(xy)) + self.structure(xy)\n",
      "            input_label_couples = [hstack([x,y]) for y in self.label_set]\n",
      "            best_label = argmax(input_label_couples, score)[-1]\n",
      "            return best_label\n",
      "        vdecision = non_scalar_vectorize(decision, (self.x_dim,), (1,1))\n",
      "        return lambda x: vdecision(x).reshape(x.shape[:-1]+(1,))\n",
      "    \n",
      "    def gradient(self, theta):\n",
      "        classif_rule = self.structured_decision(theta)\n",
      "        y_star = classif_rule(self.inputs)\n",
      "        #print \"Gradient : \"+str(y_star)\n",
      "        #print str(self.labels)\n",
      "        phi_star = self.phi(hstack([self.inputs,y_star]))\n",
      "        return mean(phi_star-self.phi_xy,axis=0)\n",
      "    \n",
      "    def run(self):\n",
      "        f_grad = lambda theta: self.gradient(theta)\n",
      "        theta = super(StructuredClassifier,self).run( f_grad, b_norm=True)\n",
      "        classif_rule = greedy_policy(theta,self.phi,self.label_set)\n",
      "        return classif_rule,theta"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 24
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "single_mu = lambda sa:feature_expectations_MC[str(sa)]\n",
      "mu_E = non_scalar_vectorize(single_mu, (2,), (729,1))\n",
      "SCIRL_MC = StructuredClassifier(D_E[:,:2], 1, mu_E, 729, ACTION_SPACE)\n",
      "void,reward_SCIRL_short = SCIRL_MC.run()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(49, 1)\n",
        "Norme du gradient : 0.0713201142104, pas : 1.5, iteration : 1"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0713201142104, pas : 1.0, iteration : 2"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0713201142104, pas : 0.75, iteration : 3"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0713201142104, pas : 0.6, iteration : 4"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0713201142104, pas : 0.5, iteration : 5"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0713201142104, pas : 0.428571428571, iteration : 6"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0713201142104, pas : 0.375, iteration : 7"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0713201142104, pas : 0.333333333333, iteration : 8"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0713201142104, pas : 0.3, iteration : 9"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0713201142104, pas : 0.272727272727, iteration : 10"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0713201142104, pas : 0.25, iteration : 11"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0713201142104, pas : 0.230769230769, iteration : 12"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0713201142104, pas : 0.214285714286, iteration : 13"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0713201142104, pas : 0.2, iteration : 14"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0713201142104, pas : 0.1875, iteration : 15"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0713201142104, pas : 0.176470588235, iteration : 16"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0713201142104, pas : 0.166666666667, iteration : 17"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0713201142104, pas : 0.157894736842, iteration : 18"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0713201142104, pas : 0.15, iteration : 19"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0713201142104, pas : 0.142857142857, iteration : 20"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0713201142104, pas : 0.136363636364, iteration : 21"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0713201142104, pas : 0.130434782609, iteration : 22"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0688846232288, pas : 0.125, iteration : 23"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0643693856023, pas : 0.12, iteration : 24"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0643693856023, pas : 0.115384615385, iteration : 25"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0643693856023, pas : 0.111111111111, iteration : 26"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0643693856023, pas : 0.107142857143, iteration : 27"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0643693856023, pas : 0.103448275862, iteration : 28"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.0643693856023, pas : 0.1, iteration : 29"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.060539751023, pas : 0.0967741935484, iteration : 30"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.060539751023, pas : 0.09375, iteration : 31"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.060539751023, pas : 0.0909090909091, iteration : 32"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.060539751023, pas : 0.0882352941176, iteration : 33"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.060539751023, pas : 0.0857142857143, iteration : 34"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.060539751023, pas : 0.0833333333333, iteration : 35"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.060539751023, pas : 0.0810810810811, iteration : 36"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.060539751023, pas : 0.0789473684211, iteration : 37"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.060539751023, pas : 0.0769230769231, iteration : 38"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.060539751023, pas : 0.075, iteration : 39"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Norme du gradient : 0.058943902436, pas : 0.0731707317073, iteration : 40"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Gradient de norme : 0.058943902436, a l'iteration : 40\n"
       ]
      }
     ],
     "prompt_number": 25
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def Sgenerator( ):\n",
      "    for v in range(0,3):\n",
      "        for x_b in range(0,9):\n",
      "            for y_r in range(0,9):\n",
      "                for x_r in range(0,3):\n",
      "                    yield [v,x_b,y_r,x_r]\n",
      "\n",
      "S = [s for s in Sgenerator()]\n",
      "\n",
      "A = range(0,5)\n",
      "\n",
      "def s_index( state ):\n",
      "    v = state[0]\n",
      "    x_b = state[1]\n",
      "    y_r = state[2]\n",
      "    x_r = state[3]\n",
      "    index = x_r + y_r*3 + x_b*3*9 + v*3*9*9\n",
      "    return index\n",
      "\n",
      "def sa_index( state, action ):\n",
      "    index = s_index(state) + action*3*9*9*3\n",
      "    return index\n",
      "\n",
      "print reward_SCIRL_short.shape\n",
      "reward_SCIRL=zeros((3645,1))\n",
      "for state in S:\n",
      "    current_indices = [sa_index(state,a) for a in ACTION_SPACE]\n",
      "    reward_SCIRL[current_indices] = reward_SCIRL_short[s_index(state)]\n",
      "print reward_SCIRL.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(729, 1)\n",
        "(3645, 1)"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 26
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "reward_SCIRL.shape\n",
      "Highway2 = MDP(P,reward_SCIRL)\n",
      "mPi_A, V_A, Pi_A = Highway2.optimal_policy()\n",
      "Highway3 = MDP(P,rand(3645,1))\n",
      "mPi_R, V_R, Pi_R = Highway3.optimal_policy()\n",
      "Highway.evaluate(mPi_A),Highway.evaluate(mPi_R),Highway.evaluate(mPi_E)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "MDP OK3\n",
        "Shape of P (3645, 729)\n",
        "Shape of R (3645, 1)\n",
        "Card of S 729\n",
        "Card of A 5\n",
        "Card of SA 3645\n",
        "Iteration 0, 1162\tactions changed."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Iteration 1, 872\tactions changed."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Iteration 2, 566\tactions changed."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Iteration 3, 318\tactions changed."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Iteration 4, 152\tactions changed."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Iteration 5, 68\tactions changed."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Iteration 6, 6\tactions changed."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Iteration 7, 0\tactions changed."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "MDP OK3"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Shape of P (3645, 729)\n",
        "Shape of R (3645, 1)\n",
        "Card of S 729\n",
        "Card of A 5\n",
        "Card of SA 3645\n",
        "Iteration 0, 1170\tactions changed."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Iteration 1, 764\tactions changed."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Iteration 2, 450\tactions changed."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Iteration 3, 210\tactions changed."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Iteration 4, 124\tactions changed."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Iteration 5, 40\tactions changed."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Iteration 6, 6\tactions changed."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Iteration 7, 0\tactions changed."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      },
      {
       "output_type": "pyout",
       "prompt_number": 28,
       "text": [
        "(7.2179650649218194, 0.53405279303991415, 7.7439104526748785)"
       ]
      }
     ],
     "prompt_number": 28
    }
   ],
   "metadata": {}
  }
 ]
}