continuous/q_network_autoencoder.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import math\n",
    "import os\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pandas import DataFrame\n",
    "import cPickle as pickle\n",
    "import math\n",
    "import copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "os.environ['CUDA_VISIBLE_DEVICES'] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>...</th>\n",
       "      <th>194</th>\n",
       "      <th>195</th>\n",
       "      <th>196</th>\n",
       "      <th>197</th>\n",
       "      <th>198</th>\n",
       "      <th>199</th>\n",
       "      <th>vaso_input</th>\n",
       "      <th>iv_input</th>\n",
       "      <th>reward</th>\n",
       "      <th>icustayid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.030952</td>\n",
       "      <td>0.066561</td>\n",
       "      <td>0.074645</td>\n",
       "      <td>0.057461</td>\n",
       "      <td>0.025694</td>\n",
       "      <td>0.018508</td>\n",
       "      <td>0.058811</td>\n",
       "      <td>0.055139</td>\n",
       "      <td>0.035398</td>\n",
       "      <td>...</td>\n",
       "      <td>0.026454</td>\n",
       "      <td>0.064301</td>\n",
       "      <td>0.026370</td>\n",
       "      <td>0.055847</td>\n",
       "      <td>0.052902</td>\n",
       "      <td>0.030440</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.031116</td>\n",
       "      <td>0.072229</td>\n",
       "      <td>0.066647</td>\n",
       "      <td>0.050890</td>\n",
       "      <td>0.038605</td>\n",
       "      <td>0.022418</td>\n",
       "      <td>0.064106</td>\n",
       "      <td>0.051121</td>\n",
       "      <td>0.035535</td>\n",
       "      <td>...</td>\n",
       "      <td>0.029728</td>\n",
       "      <td>0.060433</td>\n",
       "      <td>0.027990</td>\n",
       "      <td>0.066878</td>\n",
       "      <td>0.064350</td>\n",
       "      <td>0.048884</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.028333</td>\n",
       "      <td>0.054361</td>\n",
       "      <td>0.079503</td>\n",
       "      <td>0.056118</td>\n",
       "      <td>0.027614</td>\n",
       "      <td>0.021335</td>\n",
       "      <td>0.082840</td>\n",
       "      <td>0.061822</td>\n",
       "      <td>0.028901</td>\n",
       "      <td>...</td>\n",
       "      <td>0.038930</td>\n",
       "      <td>0.064668</td>\n",
       "      <td>0.026851</td>\n",
       "      <td>0.062974</td>\n",
       "      <td>0.065182</td>\n",
       "      <td>0.071455</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.033604</td>\n",
       "      <td>0.061950</td>\n",
       "      <td>0.078464</td>\n",
       "      <td>0.059646</td>\n",
       "      <td>0.030114</td>\n",
       "      <td>0.022200</td>\n",
       "      <td>0.075564</td>\n",
       "      <td>0.056616</td>\n",
       "      <td>0.035092</td>\n",
       "      <td>...</td>\n",
       "      <td>0.046743</td>\n",
       "      <td>0.059576</td>\n",
       "      <td>0.028979</td>\n",
       "      <td>0.064714</td>\n",
       "      <td>0.067708</td>\n",
       "      <td>0.069163</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.041468</td>\n",
       "      <td>0.083649</td>\n",
       "      <td>0.086171</td>\n",
       "      <td>0.074791</td>\n",
       "      <td>0.036465</td>\n",
       "      <td>0.036529</td>\n",
       "      <td>0.062812</td>\n",
       "      <td>0.060061</td>\n",
       "      <td>0.041216</td>\n",
       "      <td>...</td>\n",
       "      <td>0.048424</td>\n",
       "      <td>0.044860</td>\n",
       "      <td>0.037925</td>\n",
       "      <td>0.059954</td>\n",
       "      <td>0.072518</td>\n",
       "      <td>0.060507</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 205 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0         0         1         2         3         4         5  \\\n",
       "0           0  0.030952  0.066561  0.074645  0.057461  0.025694  0.018508   \n",
       "1           1  0.031116  0.072229  0.066647  0.050890  0.038605  0.022418   \n",
       "2           2  0.028333  0.054361  0.079503  0.056118  0.027614  0.021335   \n",
       "3           3  0.033604  0.061950  0.078464  0.059646  0.030114  0.022200   \n",
       "4           4  0.041468  0.083649  0.086171  0.074791  0.036465  0.036529   \n",
       "\n",
       "          6         7         8    ...           194       195       196  \\\n",
       "0  0.058811  0.055139  0.035398    ...      0.026454  0.064301  0.026370   \n",
       "1  0.064106  0.051121  0.035535    ...      0.029728  0.060433  0.027990   \n",
       "2  0.082840  0.061822  0.028901    ...      0.038930  0.064668  0.026851   \n",
       "3  0.075564  0.056616  0.035092    ...      0.046743  0.059576  0.028979   \n",
       "4  0.062812  0.060061  0.041216    ...      0.048424  0.044860  0.037925   \n",
       "\n",
       "        197       198       199  vaso_input  iv_input  reward  icustayid  \n",
       "0  0.055847  0.052902  0.030440         0.0       4.0       0          3  \n",
       "1  0.066878  0.064350  0.048884         0.0       4.0       0          3  \n",
       "2  0.062974  0.065182  0.071455         0.0       2.0       0          3  \n",
       "3  0.064714  0.067708  0.069163         0.0       2.0       0          3  \n",
       "4  0.059954  0.072518  0.060507         0.0       2.0       0          3  \n",
       "\n",
       "[5 rows x 205 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('../data/rl_train_data_final_auto.csv')\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "val = pd.read_csv(\"../data/rl_val_data_final_auto.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>...</th>\n",
       "      <th>194</th>\n",
       "      <th>195</th>\n",
       "      <th>196</th>\n",
       "      <th>197</th>\n",
       "      <th>198</th>\n",
       "      <th>199</th>\n",
       "      <th>vaso_input</th>\n",
       "      <th>iv_input</th>\n",
       "      <th>reward</th>\n",
       "      <th>icustayid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.087881</td>\n",
       "      <td>0.039770</td>\n",
       "      <td>0.115117</td>\n",
       "      <td>0.091252</td>\n",
       "      <td>0.048736</td>\n",
       "      <td>0.071957</td>\n",
       "      <td>0.054080</td>\n",
       "      <td>0.092873</td>\n",
       "      <td>0.060730</td>\n",
       "      <td>...</td>\n",
       "      <td>0.063010</td>\n",
       "      <td>0.078916</td>\n",
       "      <td>0.100621</td>\n",
       "      <td>0.074916</td>\n",
       "      <td>0.043951</td>\n",
       "      <td>0.041919</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.070638</td>\n",
       "      <td>0.044847</td>\n",
       "      <td>0.124776</td>\n",
       "      <td>0.080744</td>\n",
       "      <td>0.064025</td>\n",
       "      <td>0.074907</td>\n",
       "      <td>0.049124</td>\n",
       "      <td>0.080847</td>\n",
       "      <td>0.053200</td>\n",
       "      <td>...</td>\n",
       "      <td>0.063897</td>\n",
       "      <td>0.086132</td>\n",
       "      <td>0.107654</td>\n",
       "      <td>0.081821</td>\n",
       "      <td>0.067394</td>\n",
       "      <td>0.061855</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.064381</td>\n",
       "      <td>0.051187</td>\n",
       "      <td>0.136151</td>\n",
       "      <td>0.079168</td>\n",
       "      <td>0.069528</td>\n",
       "      <td>0.077527</td>\n",
       "      <td>0.043360</td>\n",
       "      <td>0.074929</td>\n",
       "      <td>0.055204</td>\n",
       "      <td>...</td>\n",
       "      <td>0.061498</td>\n",
       "      <td>0.089629</td>\n",
       "      <td>0.103321</td>\n",
       "      <td>0.089894</td>\n",
       "      <td>0.067459</td>\n",
       "      <td>0.058963</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.056379</td>\n",
       "      <td>0.053251</td>\n",
       "      <td>0.129211</td>\n",
       "      <td>0.067934</td>\n",
       "      <td>0.066060</td>\n",
       "      <td>0.069050</td>\n",
       "      <td>0.041161</td>\n",
       "      <td>0.071286</td>\n",
       "      <td>0.057056</td>\n",
       "      <td>...</td>\n",
       "      <td>0.061906</td>\n",
       "      <td>0.106822</td>\n",
       "      <td>0.106045</td>\n",
       "      <td>0.087400</td>\n",
       "      <td>0.081697</td>\n",
       "      <td>0.061567</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.049509</td>\n",
       "      <td>0.049661</td>\n",
       "      <td>0.118621</td>\n",
       "      <td>0.041094</td>\n",
       "      <td>0.048823</td>\n",
       "      <td>0.068882</td>\n",
       "      <td>0.032012</td>\n",
       "      <td>0.049417</td>\n",
       "      <td>0.068625</td>\n",
       "      <td>...</td>\n",
       "      <td>0.058164</td>\n",
       "      <td>0.133080</td>\n",
       "      <td>0.126716</td>\n",
       "      <td>0.078194</td>\n",
       "      <td>0.043568</td>\n",
       "      <td>0.031965</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 205 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0         0         1         2         3         4         5  \\\n",
       "0           0  0.087881  0.039770  0.115117  0.091252  0.048736  0.071957   \n",
       "1           1  0.070638  0.044847  0.124776  0.080744  0.064025  0.074907   \n",
       "2           2  0.064381  0.051187  0.136151  0.079168  0.069528  0.077527   \n",
       "3           3  0.056379  0.053251  0.129211  0.067934  0.066060  0.069050   \n",
       "4           4  0.049509  0.049661  0.118621  0.041094  0.048823  0.068882   \n",
       "\n",
       "          6         7         8    ...           194       195       196  \\\n",
       "0  0.054080  0.092873  0.060730    ...      0.063010  0.078916  0.100621   \n",
       "1  0.049124  0.080847  0.053200    ...      0.063897  0.086132  0.107654   \n",
       "2  0.043360  0.074929  0.055204    ...      0.061498  0.089629  0.103321   \n",
       "3  0.041161  0.071286  0.057056    ...      0.061906  0.106822  0.106045   \n",
       "4  0.032012  0.049417  0.068625    ...      0.058164  0.133080  0.126716   \n",
       "\n",
       "        197       198       199  vaso_input  iv_input  reward  icustayid  \n",
       "0  0.074916  0.043951  0.041919         0.0       4.0       0         14  \n",
       "1  0.081821  0.067394  0.061855         0.0       3.0       0         14  \n",
       "2  0.089894  0.067459  0.058963         0.0       3.0       0         14  \n",
       "3  0.087400  0.081697  0.061567         0.0       2.0       0         14  \n",
       "4  0.078194  0.043568  0.031965         0.0       0.0       0         14  \n",
       "\n",
       "[5 rows x 205 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = pd.read_csv('../data/rl_test_data_final_auto.csv')\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "train_orig = pd.read_csv('../data/rl_train_data_final_cont.csv')\n",
    "val_orig = pd.read_csv('../data/rl_val_data_final_cont.csv')\n",
    "test_orig = pd.read_csv('../data/rl_test_data_final_cont.csv')\n",
    "\n",
    "train['died_in_hosp'] = train_orig['died_in_hosp']\n",
    "val['died_in_hosp'] = val_orig['died_in_hosp']\n",
    "test['died_in_hosp'] = test_orig['died_in_hosp']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n"
     ]
    }
   ],
   "source": [
    "state_features = [str(i) for i in range(200)]\n",
    "# print state_features\n",
    "print len(state_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df = train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>...</th>\n",
       "      <th>195</th>\n",
       "      <th>196</th>\n",
       "      <th>197</th>\n",
       "      <th>198</th>\n",
       "      <th>199</th>\n",
       "      <th>vaso_input</th>\n",
       "      <th>iv_input</th>\n",
       "      <th>reward</th>\n",
       "      <th>icustayid</th>\n",
       "      <th>died_in_hosp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.030952</td>\n",
       "      <td>0.066561</td>\n",
       "      <td>0.074645</td>\n",
       "      <td>0.057461</td>\n",
       "      <td>0.025694</td>\n",
       "      <td>0.018508</td>\n",
       "      <td>0.058811</td>\n",
       "      <td>0.055139</td>\n",
       "      <td>0.035398</td>\n",
       "      <td>...</td>\n",
       "      <td>0.064301</td>\n",
       "      <td>0.026370</td>\n",
       "      <td>0.055847</td>\n",
       "      <td>0.052902</td>\n",
       "      <td>0.030440</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.031116</td>\n",
       "      <td>0.072229</td>\n",
       "      <td>0.066647</td>\n",
       "      <td>0.050890</td>\n",
       "      <td>0.038605</td>\n",
       "      <td>0.022418</td>\n",
       "      <td>0.064106</td>\n",
       "      <td>0.051121</td>\n",
       "      <td>0.035535</td>\n",
       "      <td>...</td>\n",
       "      <td>0.060433</td>\n",
       "      <td>0.027990</td>\n",
       "      <td>0.066878</td>\n",
       "      <td>0.064350</td>\n",
       "      <td>0.048884</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.028333</td>\n",
       "      <td>0.054361</td>\n",
       "      <td>0.079503</td>\n",
       "      <td>0.056118</td>\n",
       "      <td>0.027614</td>\n",
       "      <td>0.021335</td>\n",
       "      <td>0.082840</td>\n",
       "      <td>0.061822</td>\n",
       "      <td>0.028901</td>\n",
       "      <td>...</td>\n",
       "      <td>0.064668</td>\n",
       "      <td>0.026851</td>\n",
       "      <td>0.062974</td>\n",
       "      <td>0.065182</td>\n",
       "      <td>0.071455</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.033604</td>\n",
       "      <td>0.061950</td>\n",
       "      <td>0.078464</td>\n",
       "      <td>0.059646</td>\n",
       "      <td>0.030114</td>\n",
       "      <td>0.022200</td>\n",
       "      <td>0.075564</td>\n",
       "      <td>0.056616</td>\n",
       "      <td>0.035092</td>\n",
       "      <td>...</td>\n",
       "      <td>0.059576</td>\n",
       "      <td>0.028979</td>\n",
       "      <td>0.064714</td>\n",
       "      <td>0.067708</td>\n",
       "      <td>0.069163</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.041468</td>\n",
       "      <td>0.083649</td>\n",
       "      <td>0.086171</td>\n",
       "      <td>0.074791</td>\n",
       "      <td>0.036465</td>\n",
       "      <td>0.036529</td>\n",
       "      <td>0.062812</td>\n",
       "      <td>0.060061</td>\n",
       "      <td>0.041216</td>\n",
       "      <td>...</td>\n",
       "      <td>0.044860</td>\n",
       "      <td>0.037925</td>\n",
       "      <td>0.059954</td>\n",
       "      <td>0.072518</td>\n",
       "      <td>0.060507</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 206 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0         0         1         2         3         4         5  \\\n",
       "0           0  0.030952  0.066561  0.074645  0.057461  0.025694  0.018508   \n",
       "1           1  0.031116  0.072229  0.066647  0.050890  0.038605  0.022418   \n",
       "2           2  0.028333  0.054361  0.079503  0.056118  0.027614  0.021335   \n",
       "3           3  0.033604  0.061950  0.078464  0.059646  0.030114  0.022200   \n",
       "4           4  0.041468  0.083649  0.086171  0.074791  0.036465  0.036529   \n",
       "\n",
       "          6         7         8      ...            195       196       197  \\\n",
       "0  0.058811  0.055139  0.035398      ...       0.064301  0.026370  0.055847   \n",
       "1  0.064106  0.051121  0.035535      ...       0.060433  0.027990  0.066878   \n",
       "2  0.082840  0.061822  0.028901      ...       0.064668  0.026851  0.062974   \n",
       "3  0.075564  0.056616  0.035092      ...       0.059576  0.028979  0.064714   \n",
       "4  0.062812  0.060061  0.041216      ...       0.044860  0.037925  0.059954   \n",
       "\n",
       "        198       199  vaso_input  iv_input  reward  icustayid  died_in_hosp  \n",
       "0  0.052902  0.030440         0.0       4.0       0          3             0  \n",
       "1  0.064350  0.048884         0.0       4.0       0          3             0  \n",
       "2  0.065182  0.071455         0.0       2.0       0          3             0  \n",
       "3  0.067708  0.069163         0.0       2.0       0          3             0  \n",
       "4  0.072518  0.060507         0.0       2.0       0          3             0  \n",
       "\n",
       "[5 rows x 206 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "REWARD_THRESHOLD =15\n",
    "reg_lambda = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# PER important weights and params\n",
    "per_flag = True\n",
    "beta_start = 0.9\n",
    "df['prob'] = abs(df['reward'])\n",
    "temp = 1.0/df['prob']\n",
    "#temp[temp == float('Inf')] = 1.0\n",
    "df['imp_weight'] = pow((1.0/len(df) * temp), beta_start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "hidden_1_size = 400\n",
    "hidden_2_size = 400\n",
    "#  Q-network uses Leaky ReLU activation\n",
    "class Qnetwork():\n",
    "    def __init__(self):\n",
    "        self.phase = tf.placeholder(tf.bool)\n",
    "\n",
    "        self.num_actions = 25\n",
    "        # might CHANGE\n",
    "\n",
    "        self.input_size = len(state_features)\n",
    "\n",
    "        self.state = tf.placeholder(tf.float32, shape=[None, self.input_size],name=\"input_state\")\n",
    "\n",
    "        self.fc_1 = tf.contrib.layers.fully_connected(self.state, hidden_1_size, activation_fn=None)\n",
    "        self.fc_1_bn = tf.contrib.layers.batch_norm(self.fc_1, center=True, scale=True, is_training=self.phase)\n",
    "        self.fc_1_ac = tf.maximum(self.fc_1_bn, self.fc_1_bn*0.5)\n",
    "        self.fc_2 = tf.contrib.layers.fully_connected(self.fc_1_ac, hidden_2_size, activation_fn=None)\n",
    "        self.fc_2_bn = tf.contrib.layers.batch_norm(self.fc_2, center=True, scale=True, is_training=self.phase)\n",
    "        self.fc_2_ac = tf.maximum(self.fc_2_bn, self.fc_2_bn*0.5)\n",
    "        \n",
    "        # advantage and value streams\n",
    "        self.streamA,self.streamV = tf.split(self.fc_2_ac,2,axis=1)\n",
    "        self.AW = tf.Variable(tf.random_normal([hidden_2_size//2,self.num_actions]))\n",
    "        self.VW = tf.Variable(tf.random_normal([hidden_2_size//2,1]))\n",
    "        self.Advantage = tf.matmul(self.streamA,self.AW)\n",
    "        self.Value = tf.matmul(self.streamV,self.VW)\n",
    "        \n",
    "        #Then combine them together to get our final Q-values.\n",
    "        self.q_output = self.Value + tf.subtract(self.Advantage,tf.reduce_mean(self.Advantage,axis=1,keep_dims=True))\n",
    "     \n",
    "        self.predict = tf.argmax(self.q_output,1, name='predict') # vector of length batch size\n",
    "        \n",
    "        #Below we obtain the loss by taking the sum of squares difference between the target and predicted Q values.\n",
    "        self.targetQ = tf.placeholder(shape=[None],dtype=tf.float32)\n",
    "        self.actions = tf.placeholder(shape=[None],dtype=tf.int32)\n",
    "        self.actions_onehot = tf.one_hot(self.actions,self.num_actions,dtype=tf.float32)\n",
    "        \n",
    "        # Importance sampling weights for PER, used in network update    \n",
    "        self.imp_weights = tf.placeholder(shape=[None], dtype=tf.float32)\n",
    "        \n",
    "        # select the Q values for the actions that would be selected\n",
    "        self.Q = tf.reduce_sum(tf.multiply(self.q_output, self.actions_onehot), reduction_indices=1) # batch size x 1 vector\n",
    "        \n",
    "        # regularisation penalises the network when it produces rewards that are above the\n",
    "        # reward threshold, to ensure reasonable Q-value predictions  \n",
    "        self.reg_vector = tf.maximum(tf.abs(self.Q)-REWARD_THRESHOLD,0)\n",
    "        self.reg_term = tf.reduce_sum(self.reg_vector)\n",
    "        \n",
    "        self.abs_error = tf.abs(self.targetQ - self.Q)\n",
    "        \n",
    "        self.td_error = tf.square(self.targetQ - self.Q)\n",
    "        \n",
    "        # below is the loss when we are not using PER\n",
    "        self.old_loss = tf.reduce_mean(self.td_error)\n",
    "        \n",
    "        \n",
    "        # as in the paper, to get PER loss we weight the squared error by the importance weights\n",
    "        self.per_error = tf.multiply(self.td_error, self.imp_weights)\n",
    "\n",
    "        # total loss is a sum of PER loss and the regularisation term\n",
    "        if per_flag:\n",
    "            self.loss = tf.reduce_mean(self.per_error) + reg_lambda*self.reg_term\n",
    "        else:\n",
    "            self.loss = self.old_loss + reg_lambda*self.reg_term\n",
    "\n",
    "        self.trainer = tf.train.AdamOptimizer(learning_rate=0.0001)\n",
    "        self.update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "        with tf.control_dependencies(self.update_ops):\n",
    "        # Ensures that we execute the update_ops before performing the model update, so batchnorm works\n",
    "            self.update_model = self.trainer.minimize(self.loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# function is needed to update parameters between main and target network\n",
    "# tf_vars are the trainable variables to update, and tau is the rate at which to update\n",
    "# returns tf ops corresponding to the updates\n",
    "def update_target_graph(tf_vars,tau):\n",
    "    total_vars = len(tf_vars)\n",
    "    op_holder = []\n",
    "    for idx,var in enumerate(tf_vars[0:int(total_vars/2)]):\n",
    "        op_holder.append(tf_vars[idx+int(total_vars/2)].assign((var.value()*tau) + ((1-tau)*tf_vars[idx+int(total_vars/2)].value())))\n",
    "    return op_holder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def update_target(op_holder,sess):\n",
    "    for op in op_holder:\n",
    "        sess.run(op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# define an action mapping - how to get an id representing the action from the (iv,vaso) tuple\n",
    "action_map = {}\n",
    "count = 0\n",
    "for iv in range(5):\n",
    "    for vaso in range(5):\n",
    "        action_map[(iv,vaso)] = count\n",
    "        count += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# generates batches for the Q network - depending on train and eval_type, can select data from train/val/test sets.\n",
    "def process_batch(size, train=True, eval_type = None):\n",
    "    if not train:\n",
    "        if eval_type is None:\n",
    "            raise Exception('Provide eval_type to process_batch')\n",
    "        elif eval_type == 'train':\n",
    "            a = df.copy()\n",
    "        elif eval_type == 'val':\n",
    "            a = val_df.copy()\n",
    "        elif eval_type == 'test':\n",
    "            a = test_df.copy()\n",
    "        else:\n",
    "            raise Exception('Unknown eval_type')\n",
    "    else:\n",
    "        if per_flag:\n",
    "            # uses prioritised exp replay\n",
    "            a = df.sample(n=size, weights=df['prob'])\n",
    "        else:\n",
    "            a = df.sample(n=size)\n",
    "    states = None\n",
    "    actions = None\n",
    "    rewards = None\n",
    "    next_states = None\n",
    "    done_flags = None\n",
    "    for i in a.index:\n",
    "        cur_state = a.ix[i,state_features]\n",
    "        iv = int(a.ix[i, 'iv_input'])\n",
    "        vaso = int(a.ix[i, 'vaso_input'])\n",
    "        action = action_map[iv,vaso]\n",
    "        reward = a.ix[i,'reward']\n",
    "\n",
    "        if i != df.index[-1]:\n",
    "            # if not terminal step in trajectory             \n",
    "            if df.ix[i, 'icustayid'] == df.ix[i+1, 'icustayid']:\n",
    "                next_state = df.ix[i + 1, state_features]\n",
    "                done = 0\n",
    "            else:\n",
    "                # trajectory is finished\n",
    "                next_state = np.zeros(len(cur_state))\n",
    "                done = 1\n",
    "        else:\n",
    "            # last entry in df is the final state of that trajectory\n",
    "            next_state = np.zeros(len(cur_state))\n",
    "            done = 1\n",
    "\n",
    "        if states is None:\n",
    "            states = copy.deepcopy(cur_state)\n",
    "        else:\n",
    "            states = np.vstack((states,cur_state))\n",
    "\n",
    "        if actions is None:\n",
    "            actions = [action]\n",
    "        else:\n",
    "            actions = np.vstack((actions,action))\n",
    "\n",
    "        if rewards is None:\n",
    "            rewards = [reward]\n",
    "        else:\n",
    "            rewards = np.vstack((rewards,reward))\n",
    "\n",
    "        if next_states is None:\n",
    "            next_states = copy.deepcopy(next_state)\n",
    "        else:\n",
    "            next_states = np.vstack((next_states,next_state))\n",
    "\n",
    "        if done_flags is None:\n",
    "            done_flags = [done]\n",
    "        else:\n",
    "            done_flags = np.vstack((done_flags,done))\n",
    "    \n",
    "    return (states, np.squeeze(actions), np.squeeze(rewards), next_states, np.squeeze(done_flags), a)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "val_df = val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#  Used to run diagnostics on the train set\n",
    "phys_q_train = []\n",
    "agent_q_train = []\n",
    "phys_actions_tr = []\n",
    "agent_actions_tr = []\n",
    "def train_set_performance():\n",
    "    count = 0\n",
    "    global phys_q_train\n",
    "    global agent_q_train\n",
    "    global phys_actions\n",
    "    global agent_actions\n",
    "    phys_q_train = []\n",
    "    agent_q_train = []\n",
    "    phys_actions_tr = []\n",
    "    agent_actions_tr = []\n",
    "    for r in df.index:\n",
    "        cur_state = [df.ix[r,state_features]]\n",
    "        iv = int(df.ix[r, 'iv_input'])\n",
    "        vaso = int(df.ix[r, 'vaso_input'])\n",
    "        action = action_map[iv,vaso]\n",
    "        output_q = np.squeeze(sess.run(mainQN.q_output, feed_dict = {mainQN.state : cur_state, mainQN.phase : False}))\n",
    "        phys_q_train.append(output_q[action])\n",
    "        agent_q_train.append(max(output_q))\n",
    "        agent_actions_tr.append(np.argmax(output_q))\n",
    "        phys_actions_tr.append(action)\n",
    "        count += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def do_eval(eval_type)\n",
    "    states,actions,rewards,next_states, done_flags, _ = process_batch(size=None,train=False,eval_type=eval_type)\n",
    "\n",
    "    # firstly get the chosen actions at the next timestep\n",
    "    actions_from_q1 = sess.run(mainQN.predict,feed_dict={mainQN.state:next_states, mainQN.phase : 0})\n",
    "\n",
    "    # Q values for the next timestep from target network, as part of the Double DQN update\n",
    "    Q2 = sess.run(targetQN.q_output,feed_dict={targetQN.state:next_states, targetQN.phase : 0})\n",
    "\n",
    "    # handles the case when a trajectory is finished\n",
    "    end_multiplier = 1 - done_flags\n",
    "\n",
    "    # target Q value using Q values from target, and actions from main\n",
    "    double_q_value = Q2[range(batch_size),actions_from_q1]\n",
    "\n",
    "    # definition of target Q\n",
    "    targetQ = rewards + (gamma*double_q_value * end_multiplier)\n",
    "\n",
    "    # get the output q's, actions, and loss\n",
    "    q_output,actions_taken, loss = sess.run([mainQN.q_output,mainQN.predict, mainQN.abs_error], \\\n",
    "        feed_dict={mainQN.state:states,\n",
    "                   mainQN.targetQ:targetQ, \n",
    "                   mainQN.actions:actions,\n",
    "                   mainQN.phase:False})\n",
    "    \n",
    "    # return the relevant q values and actions\n",
    "    phys_q = q_output[range(len(q_output)), actions]\n",
    "    agent_q = q_output[range(len(q_output)), actions_taken]\n",
    "    error = np.mean(abs_error)\n",
    "    return phys_q, actions, agent_q, actions_taken, error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True  # Don't use all GPUs \n",
    "config.allow_soft_placement = True  # Enable manual control"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def do_save_results()\n",
    "    # get the chosen actions for the train, val, and test set when training is complete.\n",
    "    _, _, agent_q_train, agent_actions_train, _ = do_eval(eval_type = 'train')        \n",
    "    _, _, agent_q_val, agent_actions_val, _ = do_eval(eval_type = 'val')        \n",
    "    _, _, agent_q_test, agent_actions_test, _ = do_eval(eval_type = 'test')  \n",
    "    \n",
    "    # save everything for later - they're used in policy evaluation and when generating plots\n",
    "    with open(save_dir + 'dqn_autoencode_actions_train.p', 'wb') as f:\n",
    "        pickle.dump(agent_actions_train, f)\n",
    "    with open(save_dir + 'dqn_autoencode_actions_val.p', 'wb') as f:\n",
    "        pickle.dump(agent_actions_val, f)\n",
    "    with open(save_dir + 'dqn_autoencode_actions_test.p', 'wb') as f:\n",
    "        pickle.dump(agent_actions_test, f)\n",
    "\n",
    "    with open(save_dir + 'dqn_autoencode_q_train.p', 'wb') as f:\n",
    "        pickle.dump(agent_q_train, f)\n",
    "    with open(save_dir + 'dqn_autoencode_q_val.p', 'wb') as f:\n",
    "        pickle.dump(agent_q_val, f)\n",
    "    with open(save_dir + 'dqn_autoencode_q_test.p', 'wb') as f:\n",
    "        pickle.dump(agent_q_test, f)\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trying to load model...\n",
      "Model restored\n",
      "PER and Importance weights restored\n",
      "Init done\n",
      "Saved Model, step is 1000\n",
      "('Average loss is ', 0.23013358688354493)\n",
      "Saving PER and importance weights\n",
      "physactions  [ 0 15 12 18  0  0  0  0 10  0  0  5  0 15 10  0 10 10  0  5 20 10 10  0  0\n",
      "  0  5 10 10  0]\n",
      " chosen actions  [10 15 15 15 13  0 13 17 15 17  5 10  0 15  5 17  0  5 13  5  5  3  5 17 17\n",
      " 13  0 15  5 17]\n",
      "0.0\n",
      "7.36041\n",
      "10.2759\n"
     ]
    }
   ],
   "source": [
    "# The main training loop is here\n",
    "per_alpha = 0.6 # PER hyperparameter\n",
    "per_epsilon = 0.01 # PER hyperparameter\n",
    "batch_size = 30 #How many experiences to use for each training step.\n",
    "gamma = 0.99 #Discount factor on the target Q-values\n",
    "num_steps = 200000\n",
    "load_model = True #Whether to load a saved model.\n",
    "save_dir = \"./dqn_auto'\n",
    "save_path = \"./dqn_auto/ckpt\"#The path to save our model to.\n",
    "tau = 0.001 #Rate to update target network toward primary network\n",
    "tf.reset_default_graph()\n",
    "mainQN = Qnetwork()\n",
    "targetQN = Qnetwork()\n",
    "av_q_list = []\n",
    "save_results = False\n",
    "\n",
    "saver = tf.train.Saver(tf.global_variables())\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "trainables = tf.trainable_variables()\n",
    "\n",
    "target_ops = update_target_graph(trainables,tau)\n",
    "\n",
    "#Make a path for our model to be saved in.\n",
    "if not os.path.exists(save_dir):\n",
    "    os.makedirs(save_dir)\n",
    "\n",
    "with tf.Session(config=config) as sess:\n",
    "    if load_model == True:\n",
    "        print('Trying to load model...')\n",
    "        try:\n",
    "            restorer = tf.train.import_meta_graph(save_path + '.meta')\n",
    "            restorer.restore(sess, tf.train.latest_checkpoint(save_dir))\n",
    "            print \"Model restored\"\n",
    "        except IOError:\n",
    "            print \"No previous model found, running default init\"\n",
    "            sess.run(init)\n",
    "        try:\n",
    "            per_weights = pickle.load(open( save_dir + \"per_weights.p\", \"rb\" ))\n",
    "            imp_weights = pickle.load(open( save_dir + \"imp_weights.p\", \"rb\" ))\n",
    "            \n",
    "            # the PER weights, governing probability of sampling, and importance sampling\n",
    "            # weights for use in the gradient descent updates\n",
    "            df['prob'] = per_weights\n",
    "            df['imp_weight'] = imp_weights\n",
    "            print \"PER and Importance weights restored\"\n",
    "        except IOError:\n",
    "            print(\"No PER weights found - default being used for PER and importance sampling\")\n",
    "    else:\n",
    "        print(\"Running default init\")\n",
    "        sess.run(init)\n",
    "    print(\"Init done\")\n",
    "    for i in range(num_steps):\n",
    "        if save_results:\n",
    "            do_save_results()\n",
    "            break\n",
    "        net_loss = 0.0\n",
    "        net_q = 0.0\n",
    "        states,actions,rewards,next_states, done_flags, sampled_df = process_batch(batch_size)\n",
    "        \n",
    "        # firstly get the chosen actions at the next timestep\n",
    "        actions_from_q1 = sess.run(mainQN.predict,feed_dict={mainQN.state:next_states, mainQN.phase : 1})\n",
    "        \n",
    "        # actions chosen now, as a check\n",
    "        cur_act = sess.run(mainQN.predict,feed_dict={mainQN.state:states, mainQN.phase : 1})\n",
    "\n",
    "        # Q values for the next timestep from target network, as part of the Double DQN update\n",
    "        Q2 = sess.run(targetQN.q_output,feed_dict={targetQN.state:next_states, targetQN.phase : 1})\n",
    "\n",
    "        # handles the case when a trajectory is finished\n",
    "        end_multiplier = 1 - done_flags\n",
    "    \n",
    "        # target Q value using Q values from target, and actions from main\n",
    "        double_q_value = Q2[range(batch_size),actions_from_q1]\n",
    "        \n",
    "        # empirical hack to make the Q values never exceed the threshold - helps learning\n",
    "        double_q_value[double_q_value > REWARD_THRESHOLD] = REWARD_THRESHOLD\n",
    "        double_q_value[double_q_value < -REWARD_THRESHOLD] = -REWARD_THRESHOLD\n",
    "        \n",
    "        # definition of target Q\n",
    "        targetQ = rewards + (gamma*double_q_value * end_multiplier)\n",
    "\n",
    "        # Calculate the importance sampling weights for PER\n",
    "        imp_sampling_weights = np.array(sampled_df['imp_weight'] / float(max(df['imp_weight'])))\n",
    "        imp_sampling_weights[np.isnan(imp_sampling_weights)] = 1\n",
    "        imp_sampling_weights[imp_sampling_weights <= 0.001] = 0.001\n",
    "\n",
    "        # Train with the batch\n",
    "        _,loss, error = sess.run([mainQN.update_model,mainQN.loss, mainQN.abs_error], \\\n",
    "            feed_dict={mainQN.state:states,\n",
    "                       mainQN.targetQ:targetQ, \n",
    "                       mainQN.actions:actions,\n",
    "                       mainQN.phase:True,\n",
    "                       mainQN.imp_weights:imp_sampling_weights})\n",
    "\n",
    "        # Update target towards main network\n",
    "        update_target(target_ops,sess)\n",
    "        \n",
    "        net_loss += sum(error)\n",
    "        net_q += np.mean(targetQ)\n",
    "        \n",
    "        # Set the selection weight/prob to the abs prediction error and update the importance sampling weight\n",
    "        new_weights = pow((error + per_epsilon), per_alpha)\n",
    "        df.ix[df.index.isin(sampled_df.index), 'prob'] = new_weights\n",
    "        temp = 1.0/new_weights\n",
    "        df.ix[df.index.isin(sampled_df.index), 'imp_weight'] = pow(((1.0/len(df)) * temp), beta_start)\n",
    "        \n",
    "        if i % 1000 == 0 and i > 0:\n",
    "            saver.save(sess,save_path)\n",
    "            print(\"Saved Model, step is \" + str(i))\n",
    "            \n",
    "            av_loss = net_loss/1000.0\n",
    "            print(\"Average loss is \", av_loss)\n",
    "            net_loss = 0.0\n",
    "                        \n",
    "            print (\"Saving PER and importance weights\")\n",
    "            with open(save_dir + 'per_weights.p', 'wb') as f:\n",
    "                pickle.dump(df['prob'], f)\n",
    "            with open(save_dir + 'imp_weights.p', 'wb') as f:\n",
    "                pickle.dump(df['imp_weight'], f)\n",
    "        \n",
    "        if (i % 1000==0) and i > 0:\n",
    "            print \"physactions \", actions\n",
    "            print \" chosen actions \", cur_act\n",
    "            if i >= 1000:\n",
    "                # run an evaluation on the validation set\n",
    "                phys_q, phys_actions, agent_q, agent_actions, mean_abs_error = do_eval(eval_type = 'val')        \n",
    "                print mean_abs_error\n",
    "                print np.mean(phys_q)\n",
    "                print np.mean(agent_q)\n",
    "                break\n",
    "#     saver.save(sess,save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5     16952\n",
       "0     11339\n",
       "15     7589\n",
       "3      5600\n",
       "17     2259\n",
       "10     1693\n",
       "20     1658\n",
       "13     1429\n",
       "12       81\n",
       "11       67\n",
       "8        59\n",
       "21       41\n",
       "4        20\n",
       "14        5\n",
       "2         3\n",
       "6         1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(agent_actions).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f6bd7bc2f50>"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAF8NJREFUeJzt3X+QXeV93/H3t1KwZRxbYJI7jKRWSqy4I6zGxVug4zSz\nmAwIk4noDNHA0CC5NOo0spu027GF2w4d28zg1oTYrk1HMSoiQxGUOJEmkBAN5g7tTMCAIRY/TNli\n2UgjUGwJnPUvZvG3f9xH9fU+d/eKc1e6V9r3a2Znz/me55x97qOz+uz5ce+JzESSpG5/Z9gdkCSN\nHsNBklQxHCRJFcNBklQxHCRJFcNBklQxHCRJFcNBklQxHCRJlcXD7kBTZ511Vq5cubLRut/73vc4\n/fTT57dDpxDHpz/HaG6OT3/DGqPHH3/825n5c/3anbThsHLlSh577LFG67bbbcbHx+e3Q6cQx6c/\nx2hujk9/wxqjiPjmsbTztJIkqWI4SJIqhoMkqWI4SJIqhoMkqWI4SJIqhoMkqWI4SJIqhoMkqXLS\nvkNa82Pl1nur2sTaacZPfFckjRCPHCRJFcNBklQxHCRJFcNBklQxHCRJFcNBklQxHCRJlb7hEBHb\nI+JQRDw1o/7hiPh6RDwdEf+5q35dRExGxHMRcUlXfV2pTUbE1q76qoh4pNTviojT5uvFSZKaOZYj\nh9uAdd2FiLgQWA/8cmaeA3y61NcAVwLnlHW+EBGLImIR8HngUmANcFVpC/Ap4ObMfCdwBLh20Bcl\nSRpM33DIzIeAwzPK/wq4MTN/VNocKvX1wM7M/FFmfgOYBM4rX5OZ+UJmvgbsBNZHRADvB+4p6+8A\nLh/wNUmSBtT04zN+CfgnEXED8EPg32Xmo8Ay4OGudvtLDeDFGfXzgXcAr2TmdI/2lYjYDGwGaLVa\ntNvtRp2fmppqvO6pZmLtdFVrLcHx6cN9aG6OT3+jPkZNw2ExcCZwAfCPgLsj4hfmrVezyMxtwDaA\nsbGxHB8fb7SddrtN03VPNZtm+WylDY7PnNyH5ub49DfqY9Q0HPYDX8rMBL4SET8GzgIOACu62i0v\nNWapfwdYGhGLy9FDd3tJ0pA0vZX1T4ELASLil4DTgG8Du4ErI+JNEbEKWA18BXgUWF3uTDqNzkXr\n3SVcHgSuKNvdCOxq+mIkSfOj75FDRNwJjANnRcR+4HpgO7C93N76GrCx/Ef/dETcDTwDTANbMvP1\nsp0PAfcDi4Dtmfl0+REfBXZGxCeBJ4Bb5/H1SZIa6BsOmXnVLIv+2SztbwBu6FG/D7ivR/0FOncz\nSZJGhO+QliRVDAdJUsVwkCRVDAdJUsVwkCRVDAdJUsVwkCRVDAdJUsVwkCRVDAdJUsVwkCRVDAdJ\nUsVwkCRVDAdJUsVwkCRV+oZDRGyPiEPlwT4zl01EREbEWWU+IuKzETEZEV+LiHO72m6MiOfL18au\n+nsjYm9Z57MREfP14iRJzRzLkcNtwLqZxYhYAVwMfKurfCmdR4OuBjYDt5S2Z9J5gtz5dB7sc31E\nnFHWuQX47a71qp8lSTqx+oZDZj4EHO6x6GbgI0B21dYDt2fHw8DSiDgbuATYk5mHM/MIsAdYV5a9\nLTMfLo8ZvR24fLCXJEkaVKNrDhGxHjiQmX89Y9Ey4MWu+f2lNld9f4+6JGmI+j5DeqaIeAvwMTqn\nlE6oiNhM53QVrVaLdrvdaDtTU1ON1z3VTKydrmqtJTg+fbgPzc3x6W/Ux+gNhwPwi8Aq4K/LtePl\nwFcj4jzgALCiq+3yUjsAjM+ot0t9eY/2PWXmNmAbwNjYWI6Pj8/WdE7tdpum655qNm29t6pNrJ1m\ng+MzJ/ehuTk+/Y36GL3h00qZuTczfz4zV2bmSjqngs7NzJeA3cA15a6lC4BXM/MgcD9wcUScUS5E\nXwzcX5Z9NyIuKHcpXQPsmqfXJklq6FhuZb0T+CvgXRGxPyKunaP5fcALwCTwh8DvAGTmYeATwKPl\n6+OlRmnzxbLO/wX+vNlLkSTNl76nlTLzqj7LV3ZNJ7Bllnbbge096o8B7+7XD0nSieM7pCVJFcNB\nklQxHCRJFcNBklQxHCRJFcNBklQxHCRJFcNBklQxHCRJFcNBklQxHCRJFcNBklQxHCRJFcNBklQx\nHCRJFcNBklQ5lifBbY+IQxHxVFftv0TE1yPiaxHxJxGxtGvZdRExGRHPRcQlXfV1pTYZEVu76qsi\n4pFSvysiTpvPFyhJeuOO5cjhNmDdjNoe4N2Z+Q+A/wNcBxARa4ArgXPKOl+IiEURsQj4PHApsAa4\nqrQF+BRwc2a+EzgCzPUYUknSCdA3HDLzIeDwjNpfZuZ0mX0YWF6m1wM7M/NHmfkNOs+FPq98TWbm\nC5n5GrATWB8RAbwfuKesvwO4fMDXJEkaUN9nSB+Dfw7cVaaX0QmLo/aXGsCLM+rnA+8AXukKmu72\nlYjYDGwGaLVatNvtRh2emppqvO6pZmLtdFVrLcHx6cN9aG6OT3+jPkYDhUNE/HtgGrhjfrozt8zc\nBmwDGBsby/Hx8UbbabfbNF33VLNp671VbWLtNBscnzm5D83N8elv1MeocThExCbg14GLMjNL+QCw\noqvZ8lJjlvp3gKURsbgcPXS3lyQNSaNbWSNiHfAR4Dcy8/tdi3YDV0bEmyJiFbAa+ArwKLC63Jl0\nGp2L1rtLqDwIXFHW3wjsavZSJEnz5VhuZb0T+CvgXRGxPyKuBf4r8LPAnoh4MiL+G0BmPg3cDTwD\n/AWwJTNfL0cFHwLuB54F7i5tAT4K/NuImKRzDeLWeX2FkqQ3rO9ppcy8qkd51v/AM/MG4IYe9fuA\n+3rUX6BzN5MkaUT4DmlJUsVwkCRV5uN9DiedvQde7XkL574bLxtCbyRp9HjkIEmqGA6SpIrhIEmq\nGA6SpIrhIEmqGA6SpIrhIEmqGA6SpIrhIEmqGA6SpIrhIEmqGA6SpMqxPOxne0QcioinumpnRsSe\niHi+fD+j1CMiPhsRkxHxtYg4t2udjaX98xGxsav+3ojYW9b5bETEfL9ISdIbcyxHDrcB62bUtgIP\nZOZq4IEyD3ApnUeDrgY2A7dAJ0yA64Hz6TzY5/qjgVLa/HbXejN/liTpBOsbDpn5EHB4Rnk9sKNM\n7wAu76rfnh0PA0sj4mzgEmBPZh7OzCPAHmBdWfa2zHy4PE/69q5tSZKGpOk1h1ZmHizTLwGtMr0M\neLGr3f5Sm6u+v0ddkjREAz/sJzMzInI+OtNPRGymc7qKVqtFu91utJ3WEphYO13Vm27vZNZrHFpL\nFuZYvBFTU1OO0Rwcn/5GfYyahsPLEXF2Zh4sp4YOlfoBYEVXu+WldgAYn1Fvl/ryHu17ysxtwDaA\nsbGxHB8fn63pnD53xy5u2lu/9H1XN9veyazXE/Em1k6zoeHYLhTtdpum+99C4Pj0N+pj1PS00m7g\n6B1HG4FdXfVryl1LFwCvltNP9wMXR8QZ5UL0xcD9Zdl3I+KCcpfSNV3bkiQNSd8jh4i4k85f/WdF\nxH46dx3dCNwdEdcC3wQ2lOb3AR8AJoHvAx8EyMzDEfEJ4NHS7uOZefQi9+/QuSNqCfDn5UuSNER9\nwyEzr5pl0UU92iawZZbtbAe296g/Bry7Xz8kSSeO75CWJFUMB0lSxXCQJFUMB0lSxXCQJFUMB0lS\nxXCQJFUMB0lSxXCQJFUMB0lSxXCQJFUMB0lSxXCQJFUMB0lSxXCQJFUGCoeI+DcR8XREPBURd0bE\nmyNiVUQ8EhGTEXFXRJxW2r6pzE+W5Su7tnNdqT8XEZcM9pIkSYNqHA4RsQz418BYZr4bWARcCXwK\nuDkz3wkcAa4tq1wLHCn1m0s7ImJNWe8cYB3whYhY1LRfkqTBDXpaaTGwJCIWA28BDgLvB+4py3cA\nl5fp9WWesvyi8tzo9cDOzPxRZn6DziNGzxuwX5KkATQOh8w8AHwa+BadUHgVeBx4JTOnS7P9wLIy\nvQx4saw7Xdq/o7veYx1J0hD0fYb0bCLiDDp/9a8CXgH+J53TQsdNRGwGNgO0Wi3a7Xaj7bSWwMTa\n6aredHsns17j0FqyMMfijZiamnKM5uD49DfqY9Q4HIBfA76RmX8DEBFfAt4HLI2IxeXoYDlwoLQ/\nAKwA9pfTUG8HvtNVP6p7nZ+SmduAbQBjY2M5Pj7eqOOfu2MXN+2tX/q+q5tt72S2aeu9VW1i7TQb\nGo7tQtFut2m6/y0Ejk9/oz5Gg1xz+BZwQUS8pVw7uAh4BngQuKK02QjsKtO7yzxl+ZczM0v9ynI3\n0ypgNfCVAfolSRpQ4yOHzHwkIu4BvgpMA0/Q+av+XmBnRHyy1G4tq9wK/FFETAKH6dyhRGY+HRF3\n0wmWaWBLZr7etF+SpMENclqJzLweuH5G+QV63G2UmT8EfnOW7dwA3DBIXyRJ82egcJAkgJUzrl1N\nrJ1m09Z72XfjZUPqkQblx2dIkiqGgySpYjhIkiqGgySpYjhIkiqGgySpYjhIkiqGgySpYjhIkiqG\ngySpYjhIkiqGgySpYjhIkiqGgySpYjhIkioDhUNELI2IeyLi6xHxbET844g4MyL2RMTz5fsZpW1E\nxGcjYjIivhYR53ZtZ2Np/3xEbJz9J0qSToRBjxw+A/xFZv594JeBZ4GtwAOZuRp4oMwDXErn+dCr\ngc3ALQARcSadp8mdT+cJctcfDRRJ0nA0DoeIeDvwq5RnRGfma5n5CrAe2FGa7QAuL9Prgduz42Fg\naUScDVwC7MnMw5l5BNgDrGvaL0nS4AY5clgF/A3w3yPiiYj4YkScDrQy82Bp8xLQKtPLgBe71t9f\narPVJUlDMsgzpBcD5wIfzsxHIuIz/OQUEgCZmRGRg3SwW0RspnNKilarRbvdbrSd1pLOM25narq9\nk1mvcWgtWZhj8UZMTU05Rl1m7kdHf8cco9mN+j40SDjsB/Zn5iNl/h464fByRJydmQfLaaNDZfkB\nYEXX+stL7QAwPqPe7vUDM3MbsA1gbGwsx8fHezXr63N37OKmvfVL33d1s+2dzDbNeDA8dH6pNzQc\n24Wi3W7TdP87Fc3cjybWTnPT3sUL8nfqWI36PtT4tFJmvgS8GBHvKqWLgGeA3cDRO442ArvK9G7g\nmnLX0gXAq+X00/3AxRFxRrkQfXGpSZKGZJAjB4APA3dExGnAC8AH6QTO3RFxLfBNYENpex/wAWAS\n+H5pS2YejohPAI+Wdh/PzMMD9kuSNICBwiEznwTGeiy6qEfbBLbMsp3twPZB+nKqWtnjtA/Avhsv\nO8E9kbSQ+A5pSVLFcJAkVQwHSVLFcJAkVQwHSVLFcJAkVQwHSVLFcJAkVQwHSVLFcJAkVQwHSVLF\ncJAkVQwHSVLFcJAkVQwHSVJl4HCIiEUR8URE/FmZXxURj0TEZETcVR4ERES8qcxPluUru7ZxXak/\nFxGXDNonSdJg5uPI4XeBZ7vmPwXcnJnvBI4A15b6tcCRUr+5tCMi1gBXAucA64AvRMSieeiXJKmh\ngcIhIpYDlwFfLPMBvB+4pzTZAVxepteXecryi0r79cDOzPxRZn6DzmNEzxukX5KkwQx65PAHwEeA\nH5f5dwCvZOZ0md8PLCvTy4AXAcryV0v7/1/vsY4kaQgaP0M6In4dOJSZj0fE+Px1ac6fuRnYDNBq\ntWi3242201oCE2unq3rT7R1PvfoJ89fXXttvLRnNsRglU1NTjlGXmfvR0d8xx2h2o74PNQ4H4H3A\nb0TEB4A3A28DPgMsjYjF5ehgOXCgtD8ArAD2R8Ri4O3Ad7rqR3Wv81MycxuwDWBsbCzHx8cbdfxz\nd+zipr31S993dbPtHU+btt7bsz5ffe21/Ym102xoOLYLRbvdpun+dyqauR9NrJ3mpr2LR/J3alSM\n+j7U+LRSZl6XmcszcyWdC8pfzsyrgQeBK0qzjcCuMr27zFOWfzkzs9SvLHczrQJWA19p2i9J0uAG\nOXKYzUeBnRHxSeAJ4NZSvxX4o4iYBA7TCRQy8+mIuBt4BpgGtmTm68ehX5KkYzQv4ZCZbaBdpl+g\nx91GmflD4DdnWf8G4Ib56IskaXC+Q1qSVDEcJEkVw0GSVDEcJEkVw0GSVDEcJEkVw0GSVDEcJEkV\nw0GSVDEcJEkVw0GSVDEcJEkVw0GSVDEcJEkVw0GSVDEcJEmVxg/7iYgVwO1AC0hgW2Z+JiLOBO4C\nVgL7gA2ZeSQigs4zpj8AfB/YlJlfLdvaCPyHsulPZuaOpv2StHCsnO0Z6zdedoJ7cuoZ5MhhGpjI\nzDXABcCWiFgDbAUeyMzVwANlHuBSOs+HXg1sBm4BKGFyPXA+nSfIXR8RZwzQL0nSgBqHQ2YePPqX\nf2b+LfAssAxYDxz9y38HcHmZXg/cnh0PA0sj4mzgEmBPZh7OzCPAHmBd035JkgYXmTn4RiJWAg8B\n7wa+lZlLSz2AI5m5NCL+DLgxM/93WfYA8FFgHHhzZn6y1P8j8IPM/HSPn7OZzlEHrVbrvTt37mzU\n30OHX+XlH9T1tcve3mh7x9PeA6/2rM9XX3ttv7UEfv7M0RuLUTI1NcVb3/rWYXdjZMzcj1pL4OUf\nHP/fqeP9+3E8DWsfuvDCCx/PzLF+7RpfczgqIt4K/DHwe5n53U4edGRmRsTg6fOT7W0DtgGMjY3l\n+Ph4o+187o5d3LS3fun7rm62veNp02znVOepr722P7F2mg0Nx3ahaLfbNN3/TkUz96OJtdPctHfx\ncf+dOt6/H8fTqO9DA92tFBE/QycY7sjML5Xyy+V0EeX7oVI/AKzoWn15qc1WlyQNSeNwKKeMbgWe\nzczf71q0G9hYpjcCu7rq10THBcCrmXkQuB+4OCLOKBeiLy41SdKQDHJa6X3AbwF7I+LJUvsYcCNw\nd0RcC3wT2FCW3UfnNtZJOreyfhAgMw9HxCeAR0u7j2fm4QH6Jc2rXrdLTqydZvzEd0U6YRqHQ7mw\nHLMsvqhH+wS2zLKt7cD2pn2RpFE384+MibXTbNp678i+J8N3SEuSKgPfrSRpdPiOYc0XjxwkSRXD\nQZJUMRwkSRXDQZJUMRwkSRXDQZJUMRwkSRXDQZJUMRwkSRXDQZJUMRwkSRXDQZJUMRwkSZWRCYeI\nWBcRz0XEZERsHXZ/JGkhG4lwiIhFwOeBS4E1wFURsWa4vZKkhWskwgE4D5jMzBcy8zVgJ7B+yH2S\npAVrVB72swx4sWt+P3D+kPqik8zJ8oCbN9rPk+V16fgY9r9/dB7tPFwRcQWwLjP/RZn/LeD8zPzQ\njHabgc1l9l3Acw1/5FnAtxuuuxA4Pv05RnNzfPob1hj9vcz8uX6NRuXI4QCwomt+ean9lMzcBmwb\n9IdFxGOZOTbodk5Vjk9/jtHcHJ/+Rn2MRuWaw6PA6ohYFRGnAVcCu4fcJ0lasEbiyCEzpyPiQ8D9\nwCJge2Y+PeRuSdKCNRLhAJCZ9wH3naAfN/CpqVOc49OfYzQ3x6e/kR6jkbggLUkaLaNyzUGSNEIW\nVDj4ER39RcS+iNgbEU9GxGPD7s8oiIjtEXEoIp7qqp0ZEXsi4vny/Yxh9nGYZhmf/xQRB8p+9GRE\nfGCYfRymiFgREQ9GxDMR8XRE/G6pj/Q+tGDCwY/oeEMuzMz3jPJtdifYbcC6GbWtwAOZuRp4oMwv\nVLdRjw/AzWU/ek+5prhQTQMTmbkGuADYUv7vGel9aMGEA35EhxrKzIeAwzPK64EdZXoHcPkJ7dQI\nmWV8VGTmwcz8apn+W+BZOp8KMdL70EIKh14f0bFsSH0ZZQn8ZUQ8Xt6Rrt5amXmwTL8EtIbZmRH1\noYj4WjntNFKnTIYlIlYC/xB4hBHfhxZSOOjY/Epmnkvn9NuWiPjVYXdo1GXnlj9v+/tptwC/CLwH\nOAjcNNzuDF9EvBX4Y+D3MvO73ctGcR9aSOFwTB/RsdBl5oHy/RDwJ3ROx6n2ckScDVC+Hxpyf0ZK\nZr6cma9n5o+BP2SB70cR8TN0guGOzPxSKY/0PrSQwsGP6OgjIk6PiJ89Og1cDDw191oL1m5gY5ne\nCOwaYl9GztH/9Ip/ygLejyIigFuBZzPz97sWjfQ+tKDeBFdup/sDfvIRHTcMuUsjJSJ+gc7RAnTe\nPf8/HCOIiDuBcTqfovkycD3wp8DdwN8FvglsyMwFeVF2lvEZp3NKKYF9wL/sOr++oETErwD/C9gL\n/LiUP0bnusPI7kMLKhwkScdmIZ1WkiQdI8NBklQxHCRJFcNBklQxHCRJFcNBklQxHCRJFcNBklT5\nf9HQxbjbb93yAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f6bfc7edfd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "% matplotlib inline\n",
    "pd.Series(agent_actions).hist(bins=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get the chosen actions for the train, val, and test set when training is complete.\n",
    "_, _, agent_q_train, agent_actions_train, _ = do_eval(eval_type = 'train')        \n",
    "_, _, agent_q_val, agent_actions_val, _ = do_eval(eval_type = 'val')        \n",
    "_, _, agent_q_test, agent_actions_test, _ = do_eval(eval_type = 'test')        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# save everything for later - they're used in policy evaluation and when generating plots\n",
    "with open(save_dir + 'dqn_autoencode_actions_train.p', 'wb') as f:\n",
    "    pickle.dump(agent_actions_train, f)\n",
    "with open(save_dir + 'dqn_autoencode_actions_val.p', 'wb') as f:\n",
    "    pickle.dump(agent_actions_val, f)\n",
    "with open(save_dir + 'dqn_autoencode_actions_test.p', 'wb') as f:\n",
    "    pickle.dump(agent_actions_test, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(save_dir + 'dqn_autoencode_q_train.p', 'wb') as f:\n",
    "    pickle.dump(agent_q_train, f)\n",
    "with open(save_dir + 'dqn_autoencode_q_val.p', 'wb') as f:\n",
    "    pickle.dump(agent_q_val, f)\n",
    "with open(save_dir + 'dqn_autoencode_q_test.p', 'wb') as f:\n",
    "    pickle.dump(agent_q_test, f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}