examples/broken/BACE.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, we analyze the BACE enyzme and build machine learning models for predicting the Ki of ligands to the protein. We will use the `deepchem` library to load this data into memory, split into train/test/validation folds, build and cross-validate models, and report statistics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Automatic pdb calling has been turned OFF\n",
      "Columns of dataset: ['mol','CID','Class','Model','pIC50','MW','AlogP','HBA','HBD','RB',...]\n",
      "Number of examples in dataset: 1522\n",
      "Number of examples in crystal dataset: 25\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%pdb off\n",
    "import sys\n",
    "# This will be removed\n",
    "sys.path.append('/home/rbharath/deepchem/')\n",
    "import deepchem\n",
    "from deepchem.utils.save import load_from_disk\n",
    "\n",
    "dataset_file= \"../datasets/desc_canvas_aug30.csv\"\n",
    "dataset = load_from_disk(dataset_file)\n",
    "num_display=10\n",
    "pretty_columns = (\n",
    "    \"[\" + \",\".join([\"'%s'\" % column for column in dataset.columns.values[:num_display]])\n",
    "    + \",...]\")\n",
    "\n",
    "crystal_dataset_file = \"../datasets/crystal_desc_canvas_aug30.csv\"\n",
    "crystal_dataset = load_from_disk(crystal_dataset_file)\n",
    "\n",
    "print(\"Columns of dataset: %s\" % pretty_columns)\n",
    "print(\"Number of examples in dataset: %s\" % str(dataset.shape[0]))\n",
    "print(\"Number of examples in crystal dataset: %s\" % str(crystal_dataset.shape[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To gain a visual understanding of compounds in our dataset, let's draw them using rdkit. We define a couple of helper functions to get started."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tempfile\n",
    "from rdkit import Chem\n",
    "from rdkit.Chem import Draw\n",
    "from itertools import islice\n",
    "from IPython.display import Image, display, HTML\n",
    "\n",
    "def display_images(filenames):\n",
    "    \"\"\"Helper to pretty-print images.\"\"\"\n",
    "    imagesList=''.join(\n",
    "        [\"<img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='%s' />\"\n",
    "         % str(s) for s in sorted(filenames)])\n",
    "    display(HTML(imagesList))    \n",
    "\n",
    "def mols_to_pngs(mols, basename=\"test\"):\n",
    "    \"\"\"Helper to write RDKit mols to png files.\"\"\"\n",
    "    filenames = []\n",
    "    for i, mol in enumerate(mols):\n",
    "        filename = \"%s%d.png\" % (basename, i)\n",
    "        Draw.MolToFile(mol, filename)\n",
    "        filenames.append(filename)\n",
    "    return filenames"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we display a compound from the dataset. Note the complex ring structures and polar structures."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='dataset0.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='dataset1.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='dataset10.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='dataset11.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='dataset2.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='dataset3.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='dataset4.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='dataset5.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='dataset6.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='dataset7.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='dataset8.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='dataset9.png' />"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "num_to_display = 12\n",
    "molecules = []\n",
    "for _, data in islice(dataset.iterrows(), num_to_display):\n",
    "    molecules.append(Chem.MolFromSmiles(data[\"mol\"]))\n",
    "display_images(mols_to_pngs(molecules, basename=\"dataset\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's picture the compounds in the crystal structure collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='crystal_dataset0.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='crystal_dataset1.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='crystal_dataset10.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='crystal_dataset11.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='crystal_dataset2.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='crystal_dataset3.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='crystal_dataset4.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='crystal_dataset5.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='crystal_dataset6.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='crystal_dataset7.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='crystal_dataset8.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='crystal_dataset9.png' />"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "num_to_display = 12\n",
    "molecules = []\n",
    "for _, data in islice(crystal_dataset.iterrows(), num_to_display):\n",
    "    molecules.append(Chem.MolFromSmiles(data[\"mol\"]))\n",
    "display_images(mols_to_pngs(molecules, basename=\"crystal_dataset\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Analyzing the distribution of pIC50 values in the dataset gives us a nice spread."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEZCAYAAACTsIJzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XuYHGWZ/vHvHUIEEiFBNCAQoygYEIjoIBKVoOiiaPDI\n4gGJi/5cUUF38RJWF9BVFxQVXTztIiQqIgeFRcEVRAZZOSoECQRZxAkHTUANhAQwCfP8/qjqpKen\ne1I909X19uT+XFdf6aqu7rozfXirnrfqLUUEZmZmNROqDmBmZmlxw2BmZkO4YTAzsyHcMJiZ2RBu\nGMzMbAg3DGZmNoQbBmubpMWSXlF1jipJepOk+yQ9KmnvqvN0g6STJX236hxWPjcMNoSkAUmvapg3\nX9I1temIeEFE/HIjrzNT0qCk8foZOw04OiKeGhG3tvPExr9nPu8dkn6dNzR/lHSZpDl1yz+ZP1a7\nvaLuudtKukjSqvz9e3uL9e6XLzO5yWO3SDp6I9F90tMmYrx+aW30gs7+AKiDr7XhRaXNynjdgusW\nMAO4o0Ov90/Al4HPAM8Adga+BsyrW+xXeSNUu9U3zF8Dnsif+07gG5J2b1xPRFwP3A+8tWH9LwBm\nAeduLGpb/zHrWW4YrIghDUW+VfrK/P6++ZbuI5KWSTotX6z2w/VwvoX7EmU+mT9/uaSFkraue913\nS1oq6c91y9XWc7KkCyV9V9IjwJGS+iRdJ2lFvpX9H5I2r3u9QUkfkHSXpJWSPi1pF0nX5nnPq1++\n4f/YNKukpwCPApsBt0r6vxbPH5T0YUm/l/SQpM/nDUrjctsAnyLb+7g4Ih6PiCcj4tKI+Hj9oi3W\nMxl4M/CvEfFYRPwK+G/giGbLAwuBdzfMezdwaUSskPQVSffmf59fS3pZi/XOlXRfw7z1e5v53+94\nSXfn7+d5kqblj20h6Xv5/BWSbpT0jBZ5rQJuGKyZxh+hxun6huIrwJcjYhvgOcAF+fyX5/9uk2/h\n3gC8BzgSmJsvOwU4AyDfwv0a8HZgB2Ab4JkN650HXJCv6/vAk8CxwNOAlwKvAhrLIa8B9gH2Az4O\nfAt4B9lW+Qvy9TXTNGtE/C0ipuTL7BURz2vxfIA3Ai/K138o8A9NlnkpsAVw0QivE8AL8wbmd3mD\nVdtj2hVYFxF31y1/K7BHi9f6HvAKSTsB5KW+t5M1GAA3AnsD08j+xhdImjRCtsactc/GMWTv1yvI\n3s8VZO8vZH/XrYGdgG2B9wOPF1yHdYEbBmsk4OJ8S26FpNoXulV5aQ3wPEnb5VusN9S9TqN3Al+M\niIGIWA2cABye/8i9FbgkIq6NiLXAiU3WeW1EXAIQEU9ExM0RcWNEDEbEUuA/gQManvP5iFgVEXcA\ntwE/y9e/Evgp8MIW/69WWdv5zpwaEQ9HxH3A6TRvhJ4G/DkiBkd4nV8Ce0TE04G35K/zsfyxKcDK\nhuUfBZ7a7IXyLP1s2KN4FfAU4NL88XMiYkX+N/1S/thuI/4vm3s/8MmI+GP+fn4KeGv+Xq8h+38/\nLzK3RMSjo1iHlcQNgzUK4NCImFa7kW2Ft6ovH0W21bokLwkcMsJr7wAsrZu+F5gITM8fu399iIjH\ngb80PP/++glJu0r6iaQ/5eWlz5L94NRbXnf/8SbTU2hupKxF1Zda7mX4HhBk/8ftRmpwIuIPecNH\nRCwGPs2GfoJVZFvf9bYhaxxaWciGhuEI4NyIeBJA0nGS7pD0cL5RsA2w3Qiv1cpM4KK6jYs7gHVk\n/SDfBX4G/EDSA5JOlTRxFOuwkrhhsCJadjpGxN0R8Y58a/ZU4EJJW9J8D+OPZD8YNTPIfiyWAX8i\nKy1kK8xeo/FHvvE1v0H2g/PcvLz0CTr3mW6VdXnTpZub0XD/gSbLXAf8DXhTm/lq78ldwERJz617\nbG9g8QjPvQjYSdKB+XoXAkh6OdmeyNsiYmq+UfAIzd//1cBW68NkewJPr3v8XuDg+g2MiNgqIv4U\nEesi4tMRsQewP/B6hvd7WIXcMNiYSHqXpNoPwiNkP96DwEP5v7vULX4u8FFlh7JOAT4H/CAvo/wQ\neIOkl+Y17ZPZ+FEwU8i2jB+T9HzgA0Uit7jfaKSsRR0naaqknclq7uc1LhARj5CVzb4m6VBJW0na\nXNJrJZ0KkN+fnt9/PvBJ4OL8+auBHwGfzp/7MuANZFvlTeXPuRA4GxiIiJvzh55K1vj9WdIkSScy\nfG+k5i5gC0mvyzvwP0lWdqr5JvA5STPy3E+XNC+/P1fSnnlj8iiwlqy/yBLhhsGKGOkQ1r8DFkt6\nlOyQy8PzDtrHyEo7v8rLCfsCZ5H9YP0SuAd4DPgwQETcnt//AdnW+qPAg2Rb060yHEfWkbySrH/h\nBw3LNMvc+Hir/1fLrCO8dqP/Bn4D3AL8BPh2s/Xmtfx/IvtxfZBsa/toNnRIv5LsCKhVZH0BPyRr\nqGqOBrbMn/s94B8jYslGsi0k24v5Tt28/8lvdwEDZKW2e+seX587b9COBs4kK/GtYmjp7CvAJcDl\nklaS7Rntmz+2PdlBCo+Q7fH1M0JDZt2nsi7UI+ks4BDgwYjYM5/3BbLdxjXA74H35B8wJJ1AdtTG\nk8AxEXF5KcGsJ+Rb6SvIykRLN7Z8aiQNkmW/p+osZu0qc4/hbODghnmXkx1dsTfZVskJsP5Qxb8H\nds+f8/U2j/6wcUDSG/JyyGSyM4t/24uNglmvK+3HNyKuIdviq593RV2N9gY2dDYeSnZkxNqIGADu\nZsNup2065pF10D5A1jdxeLVxxsTDR1jPqvIQsX9gwyn4zwSur3vsfmDHrieySkXE+4D3VZ2jEyKi\nsiE7zMaqknKNpE8AayLi+yMs5i0uM7MKdH2PQdJ84HVkZ1zWPEA2REHNTjQ55luSGwszs1GIiMKD\nIHZ1j0HSwWQn0BwaEU/UPXQJ2XADkyQ9G3ge2Zgtw0REcreTTjqp8gzO5EybYi5nKnZrV2l7DJLO\nJRu3Zrt8FMaTyI5CmgRckQ80eV1EHB0Rd0g6nw2nzR8do/nfVGRgYKDqCMM4UzHOVFyKuZypHKU1\nDBHRbMCws0ZY/nMMPWnHzMwq4HMFOmD+/PlVRxjGmYpxpuJSzOVM5SjtzOcySOqlCpOZWRIkEal2\nPo9X/f39VUcYxpmKcabiUszlTOVww2BmZkO4lGRmNs65lGRmZmPihqEDUqwpOlMxzlRcirmcqRxu\nGMzMbAj3MZiZjXPuYzAzszFxw9ABKdYUnakYZyouxVzOVA43DGZmNoT7GMzMxjn3MZiZ2Zi4YeiA\nFGuKzlSMMxWXYi5nKocbBjMzG8J9DGZm45z7GMzMbEzcMHRAijVFZyrGmYpLMZczlcMNg5mZDeE+\nBjOzcc59DGZmNiZuGDogxZqiMxXjTMWlmMuZyuGGwczMhnAfg5nZOOc+BjMzGxM3DB2QYk3RmYpx\npuJSzOVM5XDDYGZmQ5TWxyDpLOAQ4MGI2DOfty1wHvAsYAA4LCIezh87AfgH4EngmIi4vMlruo/B\nzKxNKfUxnA0c3DDveOCKiNgVuDKfRtLuwN8Du+fP+bok782YmVWgtB/fiLgGWNEwex6wML+/EHhj\nfv9Q4NyIWBsRA8DdwL5lZeu0FGuKzlSMMxWXYi5nKsfELq9vekQsz+8vB6bn958JXF+33P3Ajt0M\nZjYe9c3pY9WaVcPmT5k0hZt+dVMFiawXlHoeg6SZwI/r+hhWRMS0usf/GhHbSvoP4PqIOCeffyZw\nWUT8qOH13Mdg1oZZfbPY/kPbD5u/7IxlLLlpSQWJrArt9jF0e49huaTtI2KZpB2AB/P5DwA71y23\nUz5vmPnz5zNz5kwApk6dyuzZs5k7dy6wYRfO0572dDa9euVqalYsySq702ZNSyafp8uZ7u/vZ8GC\nBQDrfy/b0e09hs8Df4mIUyUdD0yNiOPzzufvk/Ur7Aj8HHhu4+5BqnsM/f3969+cVDhTMeM9Uyf3\nGMb736pTUsyUzB6DpHOBA4DtJN0HnAicApwv6Sjyw1UBIuIOSecDdwDrgKOTbAHMzDYBHivJbBxz\nH4NBWucxmJlZD3LD0AG1Tp+UOFMxzlRcirmcqRxuGMzMbAj3MZiNY+5jMHAfg5mZjZEbhg5Isabo\nTMU4U3Ep5nKmcrhhMDOzIdzHYDaOuY/BwH0MZmY2Rm4YOiDFmqIzFeNMxaWYy5nK4YbBzMyGcB+D\n2TjmPgYD9zGYmdkYuWHogBRris5UjDMVl2IuZypHW9djkLQZMDkiVpaUx8zwtZqtWhvtY8gvuPN+\n4EngJmAb4CsR8fny4w3L4j4G2yR0qm/AfQwG5fQx7J7vIbwR+CkwEzhidPHMzCx1RRqGiZI2J2sY\nfhwRawFvttdJsaboTMU4U3Ep5nKmchRpGL5Fdn3mKcAvJc0EHikvkpmZVant8xgkCdgsItaVE2nE\ndbuPwTYJ7mOwTmq3j6HlUUmS/rlusvZrrLr7X2o/npmZpW6kUtJTycpHLwI+AOyY3/4R2Kf8aL0j\nxZqiMxXjTMWlmMuZytFyjyEiTgaQdA2wT0Q8mk+fBFzWlXRmZtZ1Rc5j+B2wd0Q8kU9vAdwaEbt1\nIV9jFvcx2CbBfQzWSR3rY6jzHeBGST8i62N4I7BwlPnMzCxxGz1cNSI+C7wHeBj4KzA/Ij5XdrBe\nkmJN0ZmKcabiUszlTOUoOlbSImBZvnxImhER95YXy8zMqlKkj+HDwEnAg2TjJQEQEXuWG61pFvcx\n2CbBfQzWSWX0MXwE2C0i/jL6WGZm1iuKDIlxL9DRYbYlfVTSYkm3Sfq+pKdI2lbSFZLuknS5pKmd\nXGeZUqwpOlMxzlRcirmcqRxF9hj+AFwl6VJgTT4vImJUZz5L2hH4MDArIv4m6TzgcGAP4IqI+Lyk\njwPH5zczM+uionsMPwcmkZ0J/dT8NhYTga0kTQS2Av4IzGPDYbALyQ6L7Qlz586tOsIwzlSMMxWX\nYi5nKsdG9xhqZ0B3SkQ8IOmLZA3O48DPIuIKSdMjYnm+2HJgeifXa2ZmxWy0YZB0VZPZERGvHM0K\nJU0j2zuYSTZ89wWS3tX44pKaHn40f/58Zs6cCcDUqVOZPXv2+ha6Vtvr9nRtXlXrbzbdmK3qPACn\nn356Eu9X/fSiRYv4yEc+kkyeeiuWrABg2qxp66d/v+T3zOqbBcDqlasBmLz1ZKZMmsIXPvuFYa9X\nW6bZ6/n9K2e6Nq/q7/+CBQsA1v9etqPI4aovrpvcAngLsC4iPtb22rLXexvwdxHx3nz6CGA/4JXA\ngRGxTNIOwFUR8fyG5yZ5uGp/f//6NycVzlRMqpk+8LEPND3M9NrjrmX/0/YfNr/V4aedPFw11b+V\nM21cxw9XjYhfN8z6X0ljuRr5UmA/SVsCTwAHATcCq4EjgVPzfy8ewzq6KrUPAThTUc5UXIq5nKkc\nRUpJ29ZNTgBeDGw92hVGxI2SLgRuBtbl//4nWYf2+ZKOIrti3GGjXYeZmY1ekaOSbgZ+k9+uA/4Z\nOGosK42IkyNiVkTsGRFHRsTaiPhrRBwUEbtGxGsi4uGxrKObGmvDKXCmYjbVTAN3DzCrb9awW9+c\nvkpztcuZylGklDSzCznMrIsGJw627HswK1JKmkR2BbdXkF3W82rgmxGxtuRsPSPFmqIzFeNMxaWY\ny5nKUeTM52/ky32N7HoMR+Tz3ltiLjMzq0iRPoa+vB/gFxFxZUTMB/YtOVdPSbGm6EzFOFNxKeZy\npnIUaRjWSXpubULSLmRHE5mZ2ThUpJT0MeAXkv6QT88ku6Kb5VKsKTpTMc5UXIq5nKkcRY5KulLS\nrsCu+azfRcTfyo1lZmZV2WgpKT9D+YPAp4CTgaMlbVFyrp6SYk3RmYpxpuJSzOVM5ShSSvoO2YV6\nvkp2VNI7gO8Cbysxl5mZVaRIw7BHROxeN/0LSXeUFagXpVhTdKZinKm4FHM5UzmKNAw3S3ppRFwH\nIGk/suExzCwRtSEuhs1fOsD2DD/D2WwkRQ5XfTHwK0lLJQ0A1wIvzq/X/NtS0/WIFGuKzlTMeMlU\nG+Ki8TYYg5XmKpszlaPIHsPBpacwM7NkFDlcdSC/6trO9ctHxM1lBuslKdYUnakYZyouxVzOVI4i\ng+j9GzAfuAeo3y89sKRMZmZWoSJ9DH8P7BIRB0TEgbVb2cF6SYo1RWcqpupMfXP6hl0TYcZuMxhY\nOlBprmaq/ls140zlKNLHsBiYBiwvOYvZJmfVmlXDrouwYskKln/bXzerTpGG4XNkh6zeDtSGwoiI\nmFderN6SYk3RmYpJMdO0WdOqjtBUin8rZypH0TOfTyXbc6j1MURpiczMrFJF+hhWR8RX8+sx9Oe3\nq0tP1kNSrCk6UzEpZlqxZEXVEZpK8W/lTOUossdwjaR/By5hQynJh6uamY1TRRqGfchKR/s1zPeR\nSbkUa4rOVEyKmdzHUJwzlaPICW5zu5DDzMwSUeR6DFMlfVnSb/LbFyVt041wvSLFmqIzFZNiJvcx\nFOdM5SjS+XwW2fUY3gYcBjwKnF1mKDMzq06RPoZdIuLNddMnS7q1rEC9KMWaojMVk2Im9zEU50zl\nKLLH8Likl9cmJL0MeKy8SGZmVqUiDcM/Al/Lr8ewFDgjn2e5FGuKzlRMipncx1CcM5WjyFFJi4C9\nah3OEfHIWFcqaSpwJrAH2aGw7wH+DzgPeBYwABwWEQ+PdV1mZtaeIkcl/bukqRHxSEQ8ImmapM+M\ncb1fAS6LiFnAXsCdwPHAFRGxK3BlPt0TUqwpOlMxKWZyH0NxzlSOIqWk19ZvuUfECuCQ0a4w3/N4\neUSclb/eunwvZB6wMF9sIfDG0a7DzMxGr0jDMEHSFrUJSVsCk8awzmcDD0k6W9LNkv5L0mRgekTU\nxhpeDkwfwzq6KsWaojMVk2Im9zEU50zlKHK46jnAlZLOAkTWH/CdMa5zH+BDEXGTpNNpKBtFREhq\nOoLr/PnzmTlzJgBTp05l9uzZ63fdam9It6drqlp/r0wvWrQoqTz9/f0sWrRoxMff/6H3M2HLbPtp\n9crVAEzeejJTJk3hC5/9wpjXX3tNGN4g1KZrpaUVS1YwuGZwxMfHunx9nvHw/lUxXVNlnv7+fhYs\nWACw/veyHYrY+Ajakl4LvCqfvCIiftb2mja81vbAdRHx7Hz6ZcAJwHOAAyNimaQdgKsi4vkNz40i\nec06ZVbfrGEX0gFYdsYylty0pLTXv/a4a9n/tP27Pr9T/y9LiyQiQkWXL7LHQET8FPjpqFMNfa1l\nku6TtGtE3AUcBNye344ku/bDkcDFnVifmZm1p0gfQxk+DJyTn0G9F/BZ4BTg1ZLuAl6ZT/eExl3I\nFDhTMSlmch9Dcc5UjkJ7DJ0WEbcCfU0eOqjbWczMbKiWewySrsz//Xz34vSmWudPSpypmBQz+TyG\n4pypHCPtMewgaX9gnqQfkB2RtL7n11dws5T1zelj1ZpVw+ZPmTSFm351UwWJeluzv6f/luPXSA3D\nScCJwI7AF5s87iu45fr7+5PbStjUM61as6rl0URVZSoqxT6GVWtW8ZRXPWXI3kzj37IKKb5/KWZq\nV8uGISIuAC6QdGJEfLqLmcysIgN3DzCrb9bw+UsH2I3dKkhkVSgyiN6nJR0KvIKslHR1RPy49GQ9\nJMWtA2cqJsVMVfYxDE4cbLqndc9x9yTZ95Hi+5dipnZttGGQdArZEUTnkPUzHCNp/4g4oexwZuNF\nqz6PgaUDbM/wH2KzKhU5j+EQ4DURcVZEfBs4GHh9ubF6S4rHLTtTMd3KVOvzaLwNxuCwZVPsY4A0\nc23Kn6kyFWkYAphaNz2VuqOTzMxsfClygtu/AzdLuoqslHQAPXSthG5IsaboTMWMNlOrTtpOHMKZ\nYi0f0sw1nj5TKSnS+XyupKvJ+hkCOD4i/lR6MrOEteqkTeEQTrOxKjRWUkT8MSL+OyIucaMwXIo1\nRWcqJsVMKdbyIc1cKb5/KWZqV1WD6JmZWaLcMHRAijVFZyomxUwp1vIhzVwpvn8pZmrXiA2DpImS\nftetMGZmVr0RG4aIWAfcKelZXcrTk1KsKTpTMSlmSrGWD2nmSvH9SzFTu4ocrrotcLukG4HaBWEj\nIuaVF8vMzKpSpGH41ybzfIJbnRRris5UTC1Tp4asGGkQuqKvk2ItH9LMlfJnqpcVOY+hX9JM4LkR\n8XNJWxV5nlkvaTVM9z3H3dPW64w0CJ1Zr9joUUmS/h9wAfCtfNZOwEVlhuo1KdYUU8rUN6ePWX2z\nmLHbDGb1zVp/65vT7Oqu3ZXS36kmxVo+pJkrxfcvxUztKrLl/0FgX+B6gIi4S9IzSk1l40pta3zF\nkhWVX+ilsdSzeuVqJm892aOcmtUp0jD8LSL+JgnIDmHFfQxDpFhTTDFTCjXqXij1pPB3aibFXCl+\nzlPM1K4iDcPVkj4BbCXp1cDRgC/UY13n6w6bdUeRM5+PBx4CbgPeD1wGfLLMUL0mxZpiipnGWqNu\ndk2DZkcSdTNTGVLMBGnmSvFznmKmdhU5KulJSQuBG8hKSHdGhEtJZmbjVJFLex4CfBOoFWGfI+n9\nEXFZqcl6SIo1xRQzpVijdqbiUsyV4uc8xUztKtLH8CXgwIi4G0DSLmTlJDcMZmbjUJE+hpW1RiF3\nD7CypDw9KcWaYoqZUqxRO1NxKeZK8XOeYqZ2tdxjkPSW/O6vJV0GnJ9Pvw34ddnBzMysGiOVkt7A\nhvMVHiS71jNkRyhtMdYVS9qMrIG5PyLeIGlb4DzgWcAAcFhEPDzW9XRDijXFFDOlWKN2puJSzJXi\n5zzFTO1q2TBExPyS130scAfw1Hz6eOCKiPi8pI/n08eXnMHMzBoUGSvpOZK+LOkiST/Ob5eMZaWS\ndgJeB5wJKJ89D1iY318IvHEs6+imFGuKKWZKsUbtTMWlmCvFz3mKmdpV5Kiki8l+wH8MDObzxnoe\nw5eBjwFb182bHhHL8/vLgeljXIeZmY1CkYbh8Yj4aqdWKOn1wIMRcYukuc2WiYiQ1LTxmT9/PjNn\nzgRg6tSpzJ49e31Nr9ZSe3ouc+fOTSZPvfqB9FavXE1/f3/h11u9cvWQ569YsoLVK2vXjhq+fG0L\nt375wTWD65dv3AIuunz945vS8rV5o33/yv58pfJ5T2G6v7+fBQsWAKz/vWyHNnYSs6R3As8FLgf+\nVpsfETe3vbbs9T4HHAGsI+vE3hr4EdAHzI2IZZJ2AK6KiOc3PNcnXfegWX2zmg5ct+yMZSy5acmY\nXqfVa7Ra57XHXcv+p+3v+R2Y3+77Z9WRRERo40tmipzH8ALgfcApwBfrbqMSEf8SETtHxLOBw4Ff\nRMQRwCXAkfliR5KVsHpCijXFFDOlWKN2puJSzJXi5zzFTO0qUko6DHhORKwpKUNtF+AU4HxJR5Ef\nrlrS+szMbARFGobbgGlkHcIdFRFXA1fn9/8KHNTpdXRDisctdyJTq+sgj3ao66LHwbdz/eWxXmM5\nxWPzU8wEaeYar9+9qhVpGKYBd0q6iQ19DBER88qLZSlodR3ksq+81s71l3vhwjtmvaZIw3BS6Sl6\nXP2RGalIMVPjpT1TkGqmFDX+rVrtrXXz4kkpfs5TzNSuItdj6O9CDjPrMa321qq4lrd1VpHrMaxi\nQwfxJGBzYFVEbN36WZuWFLcOUsyU2pY5OFM7UsyV4uc8xUztKrLHMKV2X9IEsqEr9iszlJmZVafI\neQzrRcRgRFwMHFxSnp6U4nHLKWZqrJ3XatSNt4GlA5VlSkGKmSDNXCl+zlPM1K4ipaS31E1OAF4E\nPF5aIttk+IgiszQVOSqp/roM68hOPju0rEC9KMWaYoqZUqxRO1NxKeZK8XOeYqZ2FeljmN+FHGZm\nloiWfQySTmpxO1HSid0MmboUa4opZkqxRu1MxaWYK8XPeYqZ2jXSHsNqhl93YTJwFLAd8OmyQpmZ\nWXVGurTnabX7krYGjgHeA/yAMYyuOh6lWFNMMVOKNWpnKi7FXCl+zlPM1K4R+xgkPQ34KPBO4DvA\nPhGR3v6kmZl1zEh9DKcBNwKPAntFxEluFJpLsaaYYqYUa9TOVFyKuVL8nKeYqV0jneD2T8COwCeB\nP0p6tO62sjvxzMys20bqY2jrrOhNWYo1xRQzpVijdqbiUsyV4uc8xUzt8o+/mZkN4YahA1KsKaaY\nKcUatTMVl2KuFD/nKWZqV5EhMWyca+dSmmaj1elLxVp53DB0QIo1xXYytXMpzbFIsUbtTMWNNVcZ\nl4rt9e9eqtwwmFlHtbrkp/dAe4f7GDogxZpiiplSrFE7U3FFc9WGU2+8DcZgxzOl+DlPMVO73DCY\nmdkQLiV1QIo1xTIztSoVbKwTMcXauTMVl2KuTe271y1uGKxtra68NpZORDNLh0tJHZBaTbFvTh8z\ndpsx7FrKfXP6Ks2VYu3cmYpLMVdq3z1IM1O7vMcwDq1as4pt37TtsF1/b9GbWRHeY+iAFGuKKdaD\nnamYFDNBmrlS/O6lmKldXW8YJO0s6SpJt0taLOmYfP62kq6QdJekyyVN7XY2MzOrZo9hLfDRiNgD\n2A/4oKRZwPHAFRGxK3BlPt0TUqwpVlEPrh2t1HgbWDpQWaaNcabiUsyV4ncvxUzt6nofQ0QsA5bl\n91dJWkJ23Yd5wAH5YguBfnqocbDWRyt1emgNMytXpX0MkmYCLwRuAKZHxPL8oeXA9IpitS3FmmKK\n9WBnKibFTJBmrhS/eylmaldlDYOkKcAPgWMj4tH6xyIigKgkmJnZJq6Sw1UlbU7WKHw3Ii7OZy+X\ntH1ELJO0A/Bgs+fOnz+fmTNnAjB16lRmz569voWu1fa6PV2bV9X6m+WprwfXtvRWr1xNf39/0+Xr\nn1NbfnDNICuWrFg/3Vhjbnf5+352H1NmTOno6w+uGWy5fLPpxuVX3buKnf9u58LLt/v6o1l+pOdX\nkaf2eBnvX73RfN4XLVrERz7ykVE/v4zp2rwq8/T397NgwQKA9b+X7eh6wyBJwLeBOyLi9LqHLgGO\nBE7N/724ydPX/2ebadyF69Z04xtUdZ6axl3/yVtPHrLMxpafMGnCkHmNj7e7fP2PSqdef8KkCS2X\nbzbdC8tRPRfBAAALwklEQVQ3/qBWnaemjPcP4PpvXV9oiJWqv0+99Hswd+7cIdOf+tSnaEcVewxz\ngHcBv5V0Sz7vBOAU4HxJRwEDwGEVZBuVxjcoBSnWg52pmBQzQXm5xjLESorfvRQztauKo5L+l9Z9\nGwd1M4uZmQ3nM587IMXjllM85tyZikkxE6SZK8XvXoqZ2uWGwczMhnDD0AEp1hRTrFM7UzEpZoI0\nc6X43UsxU7vcMJiZ2RBuGDogxZpiivVgZyomxUyQZq4Uv3spZmqXGwYzMxvCDUMHpFhTTLEe7EzF\npJgJ0syV4ncvxUzt8hXcekDfnD5WrVk1bH7jmaFmZp3gPYYOKLumuGrNKrb/0PbDbs0ai5oU68HO\nVEyKmSDNXCnW81PM1C43DGZmNoRLSR2QYk2xWT24doW1YfOXDrA9w8eq6UamqjlTcSnmSvG7l2Km\ndrlh2IT4CmtmVoRLSR2QYk0xxXqwMxWTYiZIJ1ffnL711xOfsduM9ff75vRVHQ1I8/egXd5jMLOe\nUjsYAxhy4Z8iw3RbMW4YOqBTNcVWh6WOpg8gxXqwMxWTYiZIM1eKmdzHYB1VvyVUz30AZtZN7mPo\ngBRriqnUg+s5UzEpZoI0c6WYKcXfg3a5YTAzsyFcSuqAuXPnNu0fKHvIipHOS9hv1n6lrXe0UqwH\nO1NxKeZKMZP7GGy9Zv0DZR8l4fMSzKwMbhg6oN2aYiePPmql/jC+VDhTMSnWzSHdv1Vqmfr7+3t+\nr8ENQwV89JGZpcwNQ5u6sbXfCaltRYEzFZViJkgzV4qZen1vAdwwtM1b+2Y23m3Sh6s++OCDPPDA\nA01va9euLfw6KdaEnakYZyouxVwpZhoP5zFs0nsMx378WBYPLGazzTcbMj8eDS694FJ22mmnMb1+\n1cNcm/Uyf3+qs0k3DE+se4ItX7Mlk585ecj8Fee0txXSqs5Z5eGkKdZenamYFDNB93MV+f6k+Ldy\nH4OZWSJa7WG0OtHU11JvLamGQdLBwOnAZsCZEXFqxZEKSbHOmeLx3c5UTIqfJ0j3b1XL1GoPo9WJ\npq0OJBnrianj4TyGZDqfJW0GnAEcDOwOvF3S8OY/QavuHb7VUTVnKsaZiksxV4qZFi1aVHWEMUum\nYQD2Be6OiIGIWAv8ADi04kyFrHtsXdURhnGmYpypuBRzpZjp4YcfrjrCmKVUStoRuK9u+n7gJRVl\n4ZA3HcIa1gyb7yMizKwMKfV5pNQwRLdXuPmEzXns2sdYu8XQcxaeXP0kj699nJ0/uvOw5zQ7ouiJ\nPz9RWsbRcqZinKm4FHOlmGlgYGBUzyurz2M0FNH13+OmJO0HnBwRB+fTJwCD9R3QktIIa2bWYyJC\nRZdNqWGYCPwOeBXwR+BG4O0RsaTSYGZmm5hkSkkRsU7Sh4CfkR2u+m03CmZm3ZfMHoOZmaUhpcNV\nW5K0s6SrJN0uabGkYxLItIWkGyQtyjOdXHWmGkmbSbpF0o+rzlIjaUDSb/NcN1adB0DSVEkXSloi\n6Y68n6vKPLvlf5/a7ZFEPusfzT/jt0n6vqSnJJDp2DzPYknHVpjjLEnLJd1WN29bSVdIukvS5ZKm\nJpDpbfnv55OS9tnYa/REwwCsBT4aEXsA+wEfrPrkt4h4AjgwImYDs4GDJVV2eG2DY4E7qOBIrxEE\nMDciXhgR+1YdJvcV4LKImAXsBVRauoyI3+V/nxcCLwIeAy6qMpOkHYEPAy+KiD3JyryHV5zpBcB7\ngT5gb+D1knapKM7ZZCfl1jseuCIidgWuzKerznQb8Cbgl0VeoCcahohYFhGL8vuryL7Az6w2FUTE\nY/ndScDmwGCFcQCQtBPwOuBMoPBRCF2STB5J2wAvj4izIOvjiohHKo5V7yDg9xFx30aXLN9EYKv8\nAJGtgAcqzvN84IaIeCIingSuBt5cRZCIuAZoHMNkHrAwv78QeGPVmSLizoi4q+hr9ETDUE/STOCF\nwA3VJgFJEyQtApYDl0dECiNvfRn4GAk0Ug0C+LmkX0t6X9VhgGcDD0k6W9LNkv5L0lZVh6pzOPD9\nqkNExAPAF4F7yY4WfDgifl5tKhYDL89LNlsBhwBjGyO/s6ZHxPL8/nJgepVhRqOnGgZJU4ALgWPz\nPYdKRcRgXkraCXiJpD2qzCPp9cCDEXELCW2d5+bkJZLXkpUCX15xnonAPsDXI2IfYDXd3+VvStIk\n4A3ABQlkmUa2BTyTbC99iqR3VpkpIu4ETgUuB34K3EJ6G0IARHZ0T0ol3UJ6pmGQtDnwQ+B7EXFx\n1Xnq5SWIqxhe1+u2/YF5kv4AnAu8UtJ3Ks4EQET8Kf/3IbK6edX9DPcD99ft5V1I1lCk4LXAb/K/\nVdUOAv4QEX+JiHXAj8g+Z5WKiLMi4sURcQDwMNk5UKlYLml7AEk7AA9WnKdtPdEwSBLwbeCOiDi9\n6jwAkrarHW0gaUvg1VTfefkvEbFzRDybrBTxi4h4d5WZACRtJemp+f3JwGvIOsMqExHLgPsk7ZrP\nOgi4vcJI9d5O1rCnYCmwn6Qt8+/hQWQHNlRK0jPyf2eQdapWXnarcwlwZH7/SCCpDVkKVBOSOcFt\nI+YA7wJ+K+mWfN4JEfE/FWbaAViYDxc+ATgvIi6rME8zqezCTgcuyn5XmAicExGXVxsJyI62OScv\n3fweeE/FeWoN50FACv0wRMSNki4EbgbW5f/+Z7WpALhQ0tPIjlg8OiJWVhFC0rnAAcB2ku4DTgRO\nAc6XdBQwABxWcaaTgL8C/wFsB1wq6ZaIeG3L1/AJbmZmVq8nSklmZtY9bhjMzGwINwxmZjaEGwYz\nMxvCDYOZmQ3hhsHMzIZww2A9RdKgpO/WTU+U9FBKQ4w3I6mtIVwk9Ut6UX5/iqRvSbo7H2vqKkn7\n5o81Hc686qGfrbe5YbBesxrYQ9IW+fSryYa36PoJOfnJjUW1m69+jJ0zgT9HxHMj4sVkJ+JtV7dc\ns+HMqx762XqYGwbrRZeRjagJG4aPEGRnDucXKrkhHzV1Xj5/pqRfSvpNfntpPn+HfP4t+YVf5uTz\n12/hS3qrpLPz+wskfVPS9cCpknaR9NN8S/6XknbLl3u2pOvyrfnPNPtP5JnulPQ9ZRcKuiAfXqV+\nmV3IxpX6ZG1eRAw0nGXfbIiDSod+tt7mhsF60XnA4cquJLYnQ4dg/wRwZUS8BHgl8IV8aOblwKsj\n4kVk40h9NV/+HcD/5CO/7g3cms+v38Jv3Np/JvDSiDiObHiID+db8h8Dvp4v8xXgaxGxF9lw1a3s\nmi+3O7ASOLruMQF7AIui9RAFrYYz7/mhn606vTJWktl6EXFbfl2OtwOXNjz8GuANko7Lp58C7Aws\nA86QtDfwJPC8/PEbgbPy0XsvjohbGVkAF0RE5MPAvxS4IB8HCrKLNkE2Aumb8vvfIxsmupn7IuK6\nuuWOIbv+Qf36RjInIv4k6enAFZLuzC/UsuEFsqwe+8YKc8NgveoS4DSywcKe3vDYmyPi/+pnKLsm\n958i4oi8b+AJyK52lV8b4vXAAklfiojvMvQHeUh5h+ySm5DtcT+c722MVv16xPA9lduBvSVNiIhh\n1xyoH85c0kVkl7u8hnzo54hY1qtDP1t1XEqyXnUWcHJENA6V/TOyrW4AJNV+tLcm22sAeDfZtYtr\nwzY/FBFnkg3tXlt+uaTnS5pAtuU/bIs7H9HzD5Lemr+WJO2VP/wrNlwbeaQL28yQtF9+/x1kP+r1\n67gH+DXwqbr/00xJr2sxnPnifLHUh362hLlhsF4TkF1yMiLOqJtX++H+N2DzvNN3MRt+UL8OHKns\nUqy7AbXO5QOBRZJuBt5G1jcA2VE8PyH7gW/sI6hvJN4JHJW/7mKyTl+AY8muVPdbsj6JVqWc3+XL\n3QFsA3yjyTLvBabnh6veRnax91q/wTX5um8AflI3nPkpwKsl3UXW13JKi/WbDeNht80qkveT/Dgi\n9qw4itkQ3mMwq5a3zCw53mMwM7MhvMdgZmZDuGEwM7Mh3DCYmdkQbhjMzGwINwxmZjaEGwYzMxvi\n/wOuCdi5tytK3wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f6d21153cd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "pIC50s = np.array(dataset[\"pIC50\"])\n",
    "# Remove some dirty data from the dataset\n",
    "pIC50s = [pIC50 for pIC50 in pIC50s if pIC50 != '']\n",
    "n, bins, patches = plt.hist(pIC50s, 50, facecolor='green', alpha=0.75)\n",
    "plt.xlabel('Measured pIC50')\n",
    "plt.ylabel('Number of compounds')\n",
    "plt.title(r'Histogram of pIC50 Values')\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now featurize the data using the Canvas samples. To do so, we must specify the columns in the data input that correspond to the features. (Note that CanvasUID is excluded!)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "user_specified_features = ['MW','AlogP','HBA','HBD','RB','HeavyAtomCount','ChiralCenterCount','ChiralCenterCountAllPossible','RingCount','PSA','Estate','MR','Polar','sLi_Key','ssBe_Key','ssssBem_Key','sBH2_Key','ssBH_Key','sssB_Key','ssssBm_Key','sCH3_Key','dCH2_Key','ssCH2_Key','tCH_Key','dsCH_Key','aaCH_Key','sssCH_Key','ddC_Key','tsC_Key','dssC_Key','aasC_Key','aaaC_Key','ssssC_Key','sNH3_Key','sNH2_Key','ssNH2_Key','dNH_Key','ssNH_Key','aaNH_Key','tN_Key','sssNH_Key','dsN_Key','aaN_Key','sssN_Key','ddsN_Key','aasN_Key','ssssN_Key','daaN_Key','sOH_Key','dO_Key','ssO_Key','aaO_Key','aOm_Key','sOm_Key','sF_Key','sSiH3_Key','ssSiH2_Key','sssSiH_Key','ssssSi_Key','sPH2_Key','ssPH_Key','sssP_Key','dsssP_Key','ddsP_Key','sssssP_Key','sSH_Key','dS_Key','ssS_Key','aaS_Key','dssS_Key','ddssS_Key','ssssssS_Key','Sm_Key','sCl_Key','sGeH3_Key','ssGeH2_Key','sssGeH_Key','ssssGe_Key','sAsH2_Key','ssAsH_Key','sssAs_Key','dsssAs_Key','ddsAs_Key','sssssAs_Key','sSeH_Key','dSe_Key','ssSe_Key','aaSe_Key','dssSe_Key','ssssssSe_Key','ddssSe_Key','sBr_Key','sSnH3_Key','ssSnH2_Key','sssSnH_Key','ssssSn_Key','sI_Key','sPbH3_Key','ssPbH2_Key','sssPbH_Key','ssssPb_Key','sLi_Cnt','ssBe_Cnt','ssssBem_Cnt','sBH2_Cnt','ssBH_Cnt','sssB_Cnt','ssssBm_Cnt','sCH3_Cnt','dCH2_Cnt','ssCH2_Cnt','tCH_Cnt','dsCH_Cnt','aaCH_Cnt','sssCH_Cnt','ddC_Cnt','tsC_Cnt','dssC_Cnt','aasC_Cnt','aaaC_Cnt','ssssC_Cnt','sNH3_Cnt','sNH2_Cnt','ssNH2_Cnt','dNH_Cnt','ssNH_Cnt','aaNH_Cnt','tN_Cnt','sssNH_Cnt','dsN_Cnt','aaN_Cnt','sssN_Cnt','ddsN_Cnt','aasN_Cnt','ssssN_Cnt','daaN_Cnt','sOH_Cnt','dO_Cnt','ssO_Cnt','aaO_Cnt','aOm_Cnt','sOm_Cnt','sF_Cnt','sSiH3_Cnt','ssSiH2_Cnt','sssSiH_Cnt','ssssSi_Cnt','sPH2_Cnt','ssPH_Cnt','sssP_Cnt','dsssP_Cnt','ddsP_Cnt','sssssP_Cnt','sSH_Cnt','dS_Cnt','ssS_Cnt','aaS_Cnt','dssS_Cnt','ddssS_Cnt','ssssssS_Cnt','Sm_Cnt','sCl_Cnt','sGeH3_Cnt','ssGeH2_Cnt','sssGeH_Cnt','ssssGe_Cnt','sAsH2_Cnt','ssAsH_Cnt','sssAs_Cnt','dsssAs_Cnt','ddsAs_Cnt','sssssAs_Cnt','sSeH_Cnt','dSe_Cnt','ssSe_Cnt','aaSe_Cnt','dssSe_Cnt','ssssssSe_Cnt','ddssSe_Cnt','sBr_Cnt','sSnH3_Cnt','ssSnH2_Cnt','sssSnH_Cnt','ssssSn_Cnt','sI_Cnt','sPbH3_Cnt','ssPbH2_Cnt','sssPbH_Cnt','ssssPb_Cnt','sLi_Sum','ssBe_Sum','ssssBem_Sum','sBH2_Sum','ssBH_Sum','sssB_Sum','ssssBm_Sum','sCH3_Sum','dCH2_Sum','ssCH2_Sum','tCH_Sum','dsCH_Sum','aaCH_Sum','sssCH_Sum','ddC_Sum','tsC_Sum','dssC_Sum','aasC_Sum','aaaC_Sum','ssssC_Sum','sNH3_Sum','sNH2_Sum','ssNH2_Sum','dNH_Sum','ssNH_Sum','aaNH_Sum','tN_Sum','sssNH_Sum','dsN_Sum','aaN_Sum','sssN_Sum','ddsN_Sum','aasN_Sum','ssssN_Sum','daaN_Sum','sOH_Sum','dO_Sum','ssO_Sum','aaO_Sum','aOm_Sum','sOm_Sum','sF_Sum','sSiH3_Sum','ssSiH2_Sum','sssSiH_Sum','ssssSi_Sum','sPH2_Sum','ssPH_Sum','sssP_Sum','dsssP_Sum','ddsP_Sum','sssssP_Sum','sSH_Sum','dS_Sum','ssS_Sum','aaS_Sum','dssS_Sum','ddssS_Sum','ssssssS_Sum','Sm_Sum','sCl_Sum','sGeH3_Sum','ssGeH2_Sum','sssGeH_Sum','ssssGe_Sum','sAsH2_Sum','ssAsH_Sum','sssAs_Sum','dsssAs_Sum','ddsAs_Sum','sssssAs_Sum','sSeH_Sum','dSe_Sum','ssSe_Sum','aaSe_Sum','dssSe_Sum','ssssssSe_Sum','ddssSe_Sum','sBr_Sum','sSnH3_Sum','ssSnH2_Sum','sssSnH_Sum','ssssSn_Sum','sI_Sum','sPbH3_Sum','ssPbH2_Sum','sssPbH_Sum','ssssPb_Sum','sLi_Avg','ssBe_Avg','ssssBem_Avg','sBH2_Avg','ssBH_Avg','sssB_Avg','ssssBm_Avg','sCH3_Avg','dCH2_Avg','ssCH2_Avg','tCH_Avg','dsCH_Avg','aaCH_Avg','sssCH_Avg','ddC_Avg','tsC_Avg','dssC_Avg','aasC_Avg','aaaC_Avg','ssssC_Avg','sNH3_Avg','sNH2_Avg','ssNH2_Avg','dNH_Avg','ssNH_Avg','aaNH_Avg','tN_Avg','sssNH_Avg','dsN_Avg','aaN_Avg','sssN_Avg','ddsN_Avg','aasN_Avg','ssssN_Avg','daaN_Avg','sOH_Avg','dO_Avg','ssO_Avg','aaO_Avg','aOm_Avg','sOm_Avg','sF_Avg','sSiH3_Avg','ssSiH2_Avg','sssSiH_Avg','ssssSi_Avg','sPH2_Avg','ssPH_Avg','sssP_Avg','dsssP_Avg','ddsP_Avg','sssssP_Avg','sSH_Avg','dS_Avg','ssS_Avg','aaS_Avg','dssS_Avg','ddssS_Avg','ssssssS_Avg','Sm_Avg','sCl_Avg','sGeH3_Avg','ssGeH2_Avg','sssGeH_Avg','ssssGe_Avg','sAsH2_Avg','ssAsH_Avg','sssAs_Avg','dsssAs_Avg','ddsAs_Avg','sssssAs_Avg','sSeH_Avg','dSe_Avg','ssSe_Avg','aaSe_Avg','dssSe_Avg','ssssssSe_Avg','ddssSe_Avg','sBr_Avg','sSnH3_Avg','ssSnH2_Avg','sssSnH_Avg','ssssSn_Avg','sI_Avg','sPbH3_Avg','ssPbH2_Avg','sssPbH_Avg','ssssPb_Avg','First Zagreb (ZM1)','First Zagreb index by valence vertex degrees (ZM1V)','Second Zagreb (ZM2)','Second Zagreb index by valence vertex degrees (ZM2V)','Polarity (Pol)','Narumi Simple Topological (NST)','Narumi Harmonic Topological (NHT)','Narumi Geometric Topological (NGT)','Total structure connectivity (TSC)','Wiener (W)','Mean Wiener (MW)','Xu (Xu)','Quadratic (QIndex)','Radial centric (RC)','Mean Square Distance Balaban (MSDB)','Superpendentic (SP)','Harary (Har)','Log of product of row sums (LPRS)','Pogliani (Pog)','Schultz Molecular Topological (SMT)','Schultz Molecular Topological by valence vertex degrees (SMTV)','Mean Distance Degree Deviation (MDDD)','Ramification (Ram)','Gutman Molecular Topological (GMT)','Gutman MTI by valence vertex degrees (GMTV)','Average vertex distance degree (AVDD)','Unipolarity (UP)','Centralization (CENT)','Variation (VAR)','Molecular electrotopological variation (MEV)','Maximal electrotopological positive variation (MEPV)','Maximal electrotopological negative variation (MENV)','Eccentric connectivity (ECCc)','Eccentricity (ECC)','Average eccentricity (AECC)','Eccentric (DECC)','Valence connectivity index chi-0 (vX0)','Valence connectivity index chi-1 (vX1)','Valence connectivity index chi-2 (vX2)','Valence connectivity index chi-3 (vX3)','Valence connectivity index chi-4 (vX4)','Valence connectivity index chi-5 (vX5)','Average valence connectivity index chi-0 (AvX0)','Average valence connectivity index chi-1 (AvX1)','Average valence connectivity index chi-2 (AvX2)','Average valence connectivity index chi-3 (AvX3)','Average valence connectivity index chi-4 (AvX4)','Average valence connectivity index chi-5 (AvX5)','Quasi Wiener (QW)','First Mohar (FM)','Second Mohar (SM)','Spanning tree number (STN)','Kier benzene-likeliness index (KBLI)','Topological charge index of order 1 (TCI1)','Topological charge index of order 2 (TCI2)','Topological charge index of order 3 (TCI3)','Topological charge index of order 4 (TCI4)','Topological charge index of order 5 (TCI5)','Topological charge index of order 6 (TCI6)','Topological charge index of order 7 (TCI7)','Topological charge index of order 8 (TCI8)','Topological charge index of order 9 (TCI9)','Topological charge index of order 10 (TCI10)','Mean topological charge index of order 1 (MTCI1)','Mean topological charge index of order 2 (MTCI2)','Mean topological charge index of order 3 (MTCI3)','Mean topological charge index of order 4 (MTCI4)','Mean topological charge index of order 5 (MTCI5)','Mean topological charge index of order 6 (MTCI6)','Mean topological charge index of order 7 (MTCI7)','Mean topological charge index of order 8 (MTCI8)','Mean topological charge index of order 9 (MTCI9)','Mean topological charge index of order 10 (MTCI10)','Global topological charge (GTC)','Hyper-distance-path index (HDPI)','Reciprocal hyper-distance-path index (RHDPI)','Square reciprocal distance sum (SRDS)','Modified Randic connectivity (MRC)','Balaban centric (BC)','Lopping centric (LC)','Kier Hall electronegativity (KHE)','Sum of topological distances between N..N (STD(N N))','Sum of topological distances between N..O (STD(N O))','Sum of topological distances between N..S (STD(N S))','Sum of topological distances between N..P (STD(N P))','Sum of topological distances between N..F (STD(N F))','Sum of topological distances between N..Cl (STD(N Cl))','Sum of topological distances between N..Br (STD(N Br))','Sum of topological distances between N..I (STD(N I))','Sum of topological distances between O..O (STD(O O))','Sum of topological distances between O..S (STD(O S))','Sum of topological distances between O..P (STD(O P))','Sum of topological distances between O..F (STD(O F))','Sum of topological distances between O..Cl (STD(O Cl))','Sum of topological distances between O..Br (STD(O Br))','Sum of topological distances between O..I (STD(O I))','Sum of topological distances between S..S (STD(S S))','Sum of topological distances between S..P (STD(S P))','Sum of topological distances between S..F (STD(S F))','Sum of topological distances between S..Cl (STD(S Cl))','Sum of topological distances between S..Br (STD(S Br))','Sum of topological distances between S..I (STD(S I))','Sum of topological distances between P..P (STD(P P))','Sum of topological distances between P..F (STD(P F))','Sum of topological distances between P..Cl (STD(P Cl))','Sum of topological distances between P..Br (STD(P Br))','Sum of topological distances between P..I (STD(P I))','Sum of topological distances between F..F (STD(F F))','Sum of topological distances between F..Cl (STD(F Cl))','Sum of topological distances between F..Br (STD(F Br))','Sum of topological distances between F..I (STD(F I))','Sum of topological distances between Cl..Cl (STD(Cl Cl))','Sum of topological distances between Cl..Br (STD(Cl Br))','Sum of topological distances between Cl..I (STD(Cl I))','Sum of topological distances between Br..Br (STD(Br Br))','Sum of topological distances between Br..I (STD(Br I))','Sum of topological distances between I..I (STD(I I))','Wiener-type index from Z weighted distance matrix - Barysz matrix (WhetZ)','Wiener-type index from electronegativity weighted distance matrix (Whete)','Wiener-type index from mass weighted distance matrix (Whetm)','Wiener-type index from van der waals weighted distance matrix (Whetv)','Wiener-type index from polarizability weighted distance matrix (Whetp)','Balaban-type index from Z weighted distance matrix - Barysz matrix (JhetZ)','Balaban-type index from electronegativity weighted distance matrix (Jhete)','Balaban-type index from mass weighted distance matrix (Jhetm)','Balaban-type index from van der waals weighted distance matrix (Jhetv)','Balaban-type index from polarizability weighted distance matrix (Jhetp)','Topological diameter (TD)','Topological radius (TR)','Petitjean 2D shape (PJ2DS)','Balaban distance connectivity index (J)','Solvation connectivity index chi-0 (SCIX0)','Solvation connectivity index chi-1 (SCIX1)','Solvation connectivity index chi-2 (SCIX2)','Solvation connectivity index chi-3 (SCIX3)','Solvation connectivity index chi-4 (SCIX4)','Solvation connectivity index chi-5 (SCIX5)','Connectivity index chi-0 (CIX0)','Connectivity chi-1 [Randic connectivity] (CIX1)','Connectivity index chi-2 (CIX2)','Connectivity index chi-3 (CIX3)','Connectivity index chi-4 (CIX4)','Connectivity index chi-5 (CIX5)','Average connectivity index chi-0 (ACIX0)','Average connectivity index chi-1 (ACIX1)','Average connectivity index chi-2 (ACIX2)','Average connectivity index chi-3 (ACIX3)','Average connectivity index chi-4 (ACIX4)','Average connectivity index chi-5 (ACIX5)','reciprocal distance Randic-type index (RDR)','reciprocal distance square Randic-type index (RDSR)','1-path Kier alpha-modified shape index (KAMS1)','2-path Kier alpha-modified shape index (KAMS2)','3-path Kier alpha-modified shape index (KAMS3)','Kier flexibility (KF)','path/walk 2 - Randic shape index (RSIpw2)','path/walk 3 - Randic shape index (RSIpw3)','path/walk 4 - Randic shape index (RSIpw4)','path/walk 5 - Randic shape index (RSIpw5)','E-state topological parameter (ETP)','Ring Count 3 (RNGCNT3)','Ring Count 4 (RNGCNT4)','Ring Count 5 (RNGCNT5)','Ring Count 6 (RNGCNT6)','Ring Count 7 (RNGCNT7)','Ring Count 8 (RNGCNT8)','Ring Count 9 (RNGCNT9)','Ring Count 10 (RNGCNT10)','Ring Count 11 (RNGCNT11)','Ring Count 12 (RNGCNT12)','Ring Count 13 (RNGCNT13)','Ring Count 14 (RNGCNT14)','Ring Count 15 (RNGCNT15)','Ring Count 16 (RNGCNT16)','Ring Count 17 (RNGCNT17)','Ring Count 18 (RNGCNT18)','Ring Count 19 (RNGCNT19)','Ring Count 20 (RNGCNT20)','Atom Count (ATMCNT)','Bond Count (BNDCNT)','Atoms in Ring System (ATMRNGCNT)','Bonds in Ring System (BNDRNGCNT)','Cyclomatic number (CYCLONUM)','Number of ring systems (NRS)','Normalized number of ring systems (NNRS)','Ring Fusion degree (RFD)','Ring perimeter (RNGPERM)','Ring bridge count (RNGBDGE)','Molecule cyclized degree (MCD)','Ring Fusion density (RFDELTA)','Ring complexity index (RCI)','Van der Waals surface area (VSA)','MR1 (MR1)','MR2 (MR2)','MR3 (MR3)','MR4 (MR4)','MR5 (MR5)','MR6 (MR6)','MR7 (MR7)','MR8 (MR8)','ALOGP1 (ALOGP1)','ALOGP2 (ALOGP2)','ALOGP3 (ALOGP3)','ALOGP4 (ALOGP4)','ALOGP5 (ALOGP5)','ALOGP6 (ALOGP6)','ALOGP7 (ALOGP7)','ALOGP8 (ALOGP8)','ALOGP9 (ALOGP9)','ALOGP10 (ALOGP10)','PEOE1 (PEOE1)','PEOE2 (PEOE2)','PEOE3 (PEOE3)','PEOE4 (PEOE4)','PEOE5 (PEOE5)','PEOE6 (PEOE6)','PEOE7 (PEOE7)','PEOE8 (PEOE8)','PEOE9 (PEOE9)','PEOE10 (PEOE10)','PEOE11 (PEOE11)','PEOE12 (PEOE12)','PEOE13 (PEOE13)','PEOE14 (PEOE14)']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tempfile, shutil\n",
    "from deepchem.featurizers.featurize import DataFeaturizer\n",
    "\n",
    "#Make directories to store the raw and featurized datasets.\n",
    "feature_dir = tempfile.mkdtemp()\n",
    "samples_dir = tempfile.mkdtemp()\n",
    "\n",
    "crystal_feature_dir = tempfile.mkdtemp()\n",
    "crystal_samples_dir = tempfile.mkdtemp()\n",
    "\n",
    "featurizer = DataFeaturizer(tasks=[\"pIC50\"],\n",
    "                            smiles_field=\"mol\",\n",
    "                            id_field=\"CID\",\n",
    "                            user_specified_features=user_specified_features,\n",
    "                            split_field=\"Model\")\n",
    "featurized_samples = featurizer.featurize(\n",
    "    dataset_file, feature_dir, samples_dir, shard_size=2000)\n",
    "\n",
    "\n",
    "crystal_featurized_samples = featurizer.featurize(\n",
    "    crystal_dataset_file, crystal_feature_dir, crystal_samples_dir, shard_size=2000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This data is already split into three subsets \"Train\" and \"Test\" with 20% and 80% respectively of the total data from the BACE enzyme. There is also a \"Validation\" set that contains data from a separate (but related assay). (Note that these names are really misnomers. The \"Test\" set would be called a validation set in standard machine-learning practice and the \"Validation\" set would typically be called an external test set.) Hence, we will rename the datasets after loading them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "splittype = \"specified\"\n",
    "train_dir = tempfile.mkdtemp()\n",
    "valid_dir = tempfile.mkdtemp()\n",
    "test_dir = tempfile.mkdtemp()\n",
    "crystal_dir = tempfile.mkdtemp()\n",
    "\n",
    "train_samples, valid_samples, test_samples = featurized_samples.train_valid_test_split(\n",
    "    splittype, train_dir, valid_dir, test_dir)\n",
    "#NOTE THE RENAMING:\n",
    "valid_samples, test_samples = test_samples, valid_samples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's quickly take a look at a compound in the validation set. (The compound displayed earlier was drawn from the train set)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='valid_set0.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='valid_set1.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='valid_set10.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='valid_set11.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='valid_set2.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='valid_set3.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='valid_set4.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='valid_set5.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='valid_set6.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='valid_set7.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='valid_set8.png' /><img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='valid_set9.png' />"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "valid_mols = [Chem.MolFromSmiles(str(compound[\"smiles\"]))\n",
    "              for compound in islice(valid_samples.itersamples(), num_to_display)]\n",
    "display_images(mols_to_pngs(valid_mols, basename=\"valid_set\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's now write these datasets to disk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of compounds in train set\n",
      "204\n",
      "Number of compounds in validation set\n",
      "1273\n",
      "Number of compounds in test set\n",
      "45\n",
      "Number of compounds in crystal set\n",
      "25\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rbharath/deepchem/deepchem/utils/dataset.py:229: UnicodeWarning: Unicode equal comparison failed to convert both arguments to Unicode - interpreting them as being unequal\n",
      "  if features[ind] == \"\":\n"
     ]
    }
   ],
   "source": [
    "from deepchem.utils.dataset import Dataset\n",
    "train_dataset = Dataset(data_dir=train_dir, samples=train_samples, \n",
    "                        featurizers=[], tasks=[\"pIC50\"],\n",
    "                        use_user_specified_features=True)\n",
    "valid_dataset = Dataset(data_dir=valid_dir, samples=valid_samples, \n",
    "                        featurizers=[], tasks=[\"pIC50\"],\n",
    "                        use_user_specified_features=True)\n",
    "test_dataset = Dataset(data_dir=test_dir, samples=test_samples, \n",
    "                       featurizers=[], tasks=[\"pIC50\"],\n",
    "                       use_user_specified_features=True)\n",
    "crystal_dataset = Dataset(data_dir=crystal_dir, samples=crystal_featurized_samples, \n",
    "                          featurizers=[], tasks=[\"pIC50\"],\n",
    "                          use_user_specified_features=True)\n",
    "print(\"Number of compounds in train set\")\n",
    "print(len(train_dataset))\n",
    "print(\"Number of compounds in validation set\")\n",
    "print(len(valid_dataset))\n",
    "print(\"Number of compounds in test set\")\n",
    "print(len(test_dataset))\n",
    "print(\"Number of compounds in crystal set\")\n",
    "print(len(crystal_dataset))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "The performance of common machine-learning algorithms can be very sensitive to preprocessing of the data. One common transformation applied to data is to normalize it to have zero-mean and unit-standard-deviation. We will apply this transformation to the pIC50 values (as seen above, the pIC50s range from 2 to 11)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from deepchem.transformers import NormalizationTransformer\n",
    "from deepchem.transformers import ClippingTransformer\n",
    "\n",
    "\n",
    "input_transformers = [NormalizationTransformer(transform_X=True, dataset=train_dataset),\n",
    "                      ClippingTransformer(transform_X=True, dataset=train_dataset)]\n",
    "output_transformers = [NormalizationTransformer(transform_y=True, dataset=train_dataset)]\n",
    "transformers = input_transformers + output_transformers\n",
    "for transformer in transformers:\n",
    "    transformer.transform(train_dataset)\n",
    "for transformer in transformers:\n",
    "    transformer.transform(valid_dataset)\n",
    "for transformer in transformers:\n",
    "    transformer.transform(test_dataset)\n",
    "for transformer in transformers:\n",
    "    transformer.transform(crystal_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now fit simple random forest models to our datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model 0/8, Metric r2_score, Validation set 0: 0.186507\n",
      "\tbest_validation_score so  far: -inf\n",
      "Model 1/8, Metric r2_score, Validation set 1: 0.176354\n",
      "\tbest_validation_score so  far: 0.186507\n",
      "Model 2/8, Metric r2_score, Validation set 2: 0.201755\n",
      "\tbest_validation_score so  far: 0.186507\n",
      "Model 3/8, Metric r2_score, Validation set 3: 0.186934\n",
      "\tbest_validation_score so  far: 0.201755\n",
      "Model 4/8, Metric r2_score, Validation set 4: 0.229487\n",
      "\tbest_validation_score so  far: 0.201755\n",
      "Model 5/8, Metric r2_score, Validation set 5: 0.269268\n",
      "\tbest_validation_score so  far: 0.229487\n",
      "Model 6/8, Metric r2_score, Validation set 6: 0.278893\n",
      "\tbest_validation_score so  far: 0.269268\n",
      "Model 7/8, Metric r2_score, Validation set 7: 0.220223\n",
      "\tbest_validation_score so  far: 0.278893\n",
      "Best hyperparameters: [('n_estimators', 100), ('data_shape', (589,)), ('max_features', 'log2')]\n",
      "train_score: 0.947125\n",
      "validation_score: 0.278893\n"
     ]
    }
   ],
   "source": [
    "from deepchem.hyperparameters import HyperparamOpt\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from deepchem.models.standard import SklearnModel\n",
    "\n",
    "\n",
    "def model_builder(task_types, params_dict, verbosity):\n",
    "    n_estimators = params_dict[\"n_estimators\"]\n",
    "    max_features = params_dict[\"max_features\"]\n",
    "    return SklearnModel(\n",
    "        task_types, params_dict,\n",
    "        model_instance=RandomForestRegressor(n_estimators=n_estimators,\n",
    "                                             max_features=max_features))\n",
    "params_dict = {\n",
    "    \"n_estimators\": [10, 100],\n",
    "    \"data_shape\": [train_dataset.get_data_shape()],\n",
    "    \"max_features\": [\"auto\", \"sqrt\", \"log2\", None],\n",
    "    }\n",
    "optimizer = HyperparamOpt(model_builder, {\"pIC50\": \"regression\"})\n",
    "best_rf, best_rf_hyperparams, all_rf_results = optimizer.hyperparam_search(\n",
    "    params_dict, train_dataset, valid_dataset, output_transformers, metric=\"r2_score\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n",
      "ERROR (theano.sandbox.cuda): Failed to compile cuda_ndarray.cu: libcublas.so.7.0: cannot open shared object file: No such file or directory\n",
      "ERROR:theano.sandbox.cuda:Failed to compile cuda_ndarray.cu: libcublas.so.7.0: cannot open shared object file: No such file or directory\n",
      "WARNING (theano.sandbox.cuda): CUDA is installed, but device gpu is not available  (error: cuda unavilable)\n",
      "WARNING:theano.sandbox.cuda:CUDA is installed, but device gpu is not available  (error: cuda unavilable)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model 0/25, Metric r2_score, Validation set 0: 0.037620\n",
      "\tbest_validation_score so  far: -inf\n",
      "Model 1/25, Metric r2_score, Validation set 1: 0.121623\n",
      "\tbest_validation_score so  far: 0.037620\n",
      "Model 2/25, Metric r2_score, Validation set 2: 0.253740\n",
      "\tbest_validation_score so  far: 0.121623\n",
      "Model 3/25, Metric r2_score, Validation set 3: 0.260108\n",
      "\tbest_validation_score so  far: 0.253740\n",
      "Model 4/25, Metric r2_score, Validation set 4: 0.219395\n",
      "\tbest_validation_score so  far: 0.260108\n",
      "Model 5/25, Metric r2_score, Validation set 5: -0.256456\n",
      "\tbest_validation_score so  far: 0.260108\n",
      "Model 6/25, Metric r2_score, Validation set 6: 0.314359\n",
      "\tbest_validation_score so  far: 0.260108\n",
      "Model 7/25, Metric r2_score, Validation set 7: 0.211534\n",
      "\tbest_validation_score so  far: 0.314359\n",
      "Model 8/25, Metric r2_score, Validation set 8: 0.210707\n",
      "\tbest_validation_score so  far: 0.314359\n",
      "Model 9/25, Metric r2_score, Validation set 9: 0.193160\n",
      "\tbest_validation_score so  far: 0.314359\n",
      "Model 10/25, Metric r2_score, Validation set 10: 0.230064\n",
      "\tbest_validation_score so  far: 0.314359\n",
      "Model 11/25, Metric r2_score, Validation set 11: 0.151662\n",
      "\tbest_validation_score so  far: 0.314359\n",
      "Model 12/25, Metric r2_score, Validation set 12: 0.288950\n",
      "\tbest_validation_score so  far: 0.314359\n",
      "Model 13/25, Metric r2_score, Validation set 13: 0.231062\n",
      "\tbest_validation_score so  far: 0.314359\n",
      "Model 14/25, Metric r2_score, Validation set 14: 0.253135\n",
      "\tbest_validation_score so  far: 0.314359\n",
      "Model 15/25, Metric r2_score, Validation set 15: 0.322635\n",
      "\tbest_validation_score so  far: 0.314359\n",
      "Model 16/25, Metric r2_score, Validation set 16: 0.010491\n",
      "\tbest_validation_score so  far: 0.322635\n",
      "Model 17/25, Metric r2_score, Validation set 17: 0.212085\n",
      "\tbest_validation_score so  far: 0.322635\n",
      "Model 18/25, Metric r2_score, Validation set 18: 0.241575\n",
      "\tbest_validation_score so  far: 0.322635\n",
      "Model 19/25, Metric r2_score, Validation set 19: 0.262623\n",
      "\tbest_validation_score so  far: 0.322635\n",
      "Model 20/25, Metric r2_score, Validation set 20: -0.002427\n",
      "\tbest_validation_score so  far: 0.322635\n",
      "Model 21/25, Metric r2_score, Validation set 21: 0.059154\n",
      "\tbest_validation_score so  far: 0.322635\n",
      "Model 22/25, Metric r2_score, Validation set 22: 0.210833\n",
      "\tbest_validation_score so  far: 0.322635\n",
      "Model 23/25, Metric r2_score, Validation set 23: 0.227074\n",
      "\tbest_validation_score so  far: 0.322635\n",
      "Model 24/25, Metric r2_score, Validation set 24: 0.244262\n",
      "\tbest_validation_score so  far: 0.322635\n",
      "Best hyperparameters: [('data_shape', (589,)), ('activation', 'relu'), ('batch_size', 50), ('nb_layers', 1), ('decay', 5.488026974514512e-06), ('nesterov', False), ('nb_epoch', 40), ('learning_rate', 0.001576528348514845), ('init', 'glorot_uniform'), ('nb_hidden', 1000), ('batchnorm', False), ('dropout', 0.5), ('momentum', 0.9)]\n",
      "train_score: 0.625197\n",
      "validation_score: 0.322635\n"
     ]
    }
   ],
   "source": [
    "from deepchem.models.deep import SingleTaskDNN\n",
    "import numpy.random\n",
    "\n",
    "np.random.seed()\n",
    "params_dict = {\"activation\": [\"relu\"],\n",
    "                \"momentum\": [.9],\n",
    "                \"batch_size\": [50],\n",
    "                \"init\": [\"glorot_uniform\"],\n",
    "                \"data_shape\": [train_dataset.get_data_shape()],\n",
    "                \"learning_rate\": np.power(10., np.random.uniform(-5, -2, size=5)),\n",
    "                \"decay\": np.power(10., np.random.uniform(-6, -4, size=5)),\n",
    "                \"nb_hidden\": [1000],\n",
    "                \"nb_epoch\": [40],\n",
    "                \"nesterov\": [False],\n",
    "                \"dropout\": [.5],\n",
    "                \"nb_layers\": [1],\n",
    "                \"batchnorm\": [False],\n",
    "              }\n",
    "\n",
    "optimizer = HyperparamOpt(SingleTaskDNN, {\"pIC50\": \"regression\"})\n",
    "best_dnn, best_hyperparams, all_results = optimizer.hyperparam_search(\n",
    "    params_dict, train_dataset, valid_dataset, output_transformers, metric=\"r2_score\", verbosity=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Before evaluating the models, let's load the collection of datapoints drawn from crystal structures."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's evaluate the best model on the validation and test sets and save the results to csv."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RF Train set R^2 0.947125\n",
      "RF Valid set R^2 0.278893\n",
      "RF Test set R^2 0.469505\n",
      "RF Crystal set R^2 0.000000\n"
     ]
    }
   ],
   "source": [
    "from deepchem.utils.evaluate import Evaluator\n",
    "\n",
    "rf_train_csv_out = \"rf_train_regressor.csv\"\n",
    "rf_train_stats_out = \"rf_train_stats_regressor.txt\"\n",
    "rf_train_evaluator = Evaluator(best_rf, train_dataset, output_transformers)\n",
    "rf_train_df, rf_train_r2score = rf_train_evaluator.compute_model_performance(\n",
    "    rf_train_csv_out, rf_train_stats_out)\n",
    "rf_train_r2_score = rf_train_r2score.iloc[0][\"r2_score\"]\n",
    "print(\"RF Train set R^2 %f\" % (rf_train_r2_score))\n",
    "\n",
    "rf_valid_csv_out = \"rf_valid_regressor.csv\"\n",
    "rf_valid_stats_out = \"rf_valid_stats_regressor.txt\"\n",
    "rf_valid_evaluator = Evaluator(best_rf, valid_dataset, output_transformers)\n",
    "rf_valid_df, rf_valid_r2score = rf_valid_evaluator.compute_model_performance(\n",
    "    rf_valid_csv_out, rf_valid_stats_out)\n",
    "rf_valid_r2_score = rf_valid_r2score.iloc[0][\"r2_score\"]\n",
    "print(\"RF Valid set R^2 %f\" % (rf_valid_r2_score))\n",
    "\n",
    "rf_test_csv_out = \"rf_test_regressor.csv\"\n",
    "rf_test_stats_out = \"rf_test_stats_regressor.txt\"\n",
    "rf_test_evaluator = Evaluator(best_rf, test_dataset, output_transformers)\n",
    "rf_test_df, rf_test_r2score = rf_test_evaluator.compute_model_performance(\n",
    "    rf_test_csv_out, rf_test_stats_out)\n",
    "rf_test_r2_score = rf_test_r2score.iloc[0][\"r2_score\"]\n",
    "print(\"RF Test set R^2 %f\" % (rf_test_r2_score))\n",
    "\n",
    "rf_crystal_csv_out = \"rf_crystal_regressor.csv\"\n",
    "rf_crystal_stats_out = \"rf_crystal_stats_regressor.txt\"\n",
    "rf_crystal_evaluator = Evaluator(best_rf, crystal_dataset, output_transformers)\n",
    "rf_crystal_df, rf_crystal_r2score = rf_crystal_evaluator.compute_model_performance(\n",
    "    rf_crystal_csv_out, rf_crystal_stats_out)\n",
    "rf_crystal_r2_score = rf_crystal_r2score.iloc[0][\"r2_score\"]\n",
    "print(\"RF Crystal set R^2 %f\" % (rf_crystal_r2_score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DNN Train set R^2 0.625197\n",
      "DNN Valid set R^2 0.322635\n",
      "DNN Test set R^2 0.271734\n",
      "DNN Crystal set R^2 0.000000\n"
     ]
    }
   ],
   "source": [
    "dnn_train_csv_out = \"dnn_train_regressor.csv\"\n",
    "dnn_train_stats_out = \"dnn_train_regressor_stats.txt\"\n",
    "dnn_train_evaluator = Evaluator(best_dnn, train_dataset, output_transformers)\n",
    "dnn_train_df, dnn_train_r2score = dnn_train_evaluator.compute_model_performance(\n",
    "    dnn_train_csv_out, dnn_train_stats_out)\n",
    "dnn_train_r2_score = dnn_train_r2score.iloc[0][\"r2_score\"]\n",
    "print(\"DNN Train set R^2 %f\" % (dnn_train_r2_score))\n",
    "\n",
    "dnn_valid_csv_out = \"dnn_valid_regressor.csv\"\n",
    "dnn_valid_stats_out = \"dnn_valid_regressor_stats.txt\"\n",
    "dnn_valid_evaluator = Evaluator(best_dnn, valid_dataset, output_transformers)\n",
    "dnn_valid_df, dnn_valid_r2score = dnn_valid_evaluator.compute_model_performance(\n",
    "    dnn_valid_csv_out, dnn_valid_stats_out)\n",
    "dnn_valid_r2_score = dnn_valid_r2score.iloc[0][\"r2_score\"]\n",
    "print(\"DNN Valid set R^2 %f\" % (dnn_valid_r2_score))\n",
    "\n",
    "dnn_test_csv_out = \"dnn_test_regressor.csv\"\n",
    "dnn_test_stats_out = \"dnn_test_regressor_stats.txt\"\n",
    "dnn_test_evaluator = Evaluator(best_dnn, test_dataset, output_transformers)\n",
    "dnn_test_df, dnn_test_r2score = dnn_test_evaluator.compute_model_performance(\n",
    "    dnn_test_csv_out, dnn_test_stats_out)\n",
    "dnn_test_r2_score = dnn_test_r2score.iloc[0][\"r2_score\"]\n",
    "print(\"DNN Test set R^2 %f\" % (dnn_test_r2_score))\n",
    "\n",
    "dnn_crystal_csv_out = \"dnn_crystal_regressor.csv\"\n",
    "dnn_crystal_stats_out = \"dnn_crystal_stats_regressor.txt\"\n",
    "dnn_crystal_evaluator = Evaluator(best_dnn, crystal_dataset, output_transformers)\n",
    "dnn_crystal_df, dnn_crystal_r2score = dnn_crystal_evaluator.compute_model_performance(\n",
    "    dnn_crystal_csv_out, dnn_crystal_stats_out)\n",
    "dnn_crystal_r2_score = dnn_crystal_r2score.iloc[0][\"r2_score\"]\n",
    "print(\"DNN Crystal set R^2 %f\" % (dnn_crystal_r2_score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rbharath/anaconda/lib/python2.7/site-packages/matplotlib/collections.py:590: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  if self._edgecolors == str('face'):\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEZCAYAAACervI0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmcHHWd//HXmyMy4ZAjCCyIoJwiIiBHRGQUZoIuwpKI\nB6sERbPuesGOCroe47qiYPBY2VVBIUGBBcGwqDCToIyQVUAhKMgvHuEKiOG+w5Hk8/ujvpNUhjm6\ne7q7qqffz8ejH1PdVV31mZ6e+t7fryICMzNrP+sUHYCZmRXDCYCZWZtyAmBm1qacAJiZtSknAGZm\nbcoJgJlZm3ICYKtJWiXp5Wn725I+04RrHi/p2kZfxyojqVfSD4qOw5rDCUCDSbpT0tOSnpB0n6Rz\nJW2Y2z9H0rNp/+DjmCJjBoiIf46I/xjrOEkDkk5oRAySdkiJ0jq51/aXdIWkRyQ9JOl6SccPOT7/\nWf7bkHOeJunB9PhKI+IejaSjJN0s6TFJD0j6uaQdmh3HKJo+MCglOqsk7d/sa7c7JwCNF8AREbEx\n8Bpgb+BTQ/afFhEb5x4/Gu9FJa073nNUqGk3DElTgZ8DVwOviIgtgH8GDh9y6Ca5z/JLuff/E3AU\n8Or0eGt6rSkk7QTMBU6KiBcDOwL/BaxsVgyNlE+oq3iPgOOAh9JPayInAE0UEcuA+WQJQdVSLukj\nkpak3OPp6R9osCrl/yR9TdKDwOclTZI0W9Jdkv6WqnU2yJ3vE5L+KukeSe8bcq05kr6Ye57Puf5F\n0jRJXwIOBs5Mue3/TMfuJmlByqEvzpdoJG0h6fJ0nuuBV1TxEXwVmBMRX42IhwEi4qaIeOeQ40b6\nXs8EZkfEXyPir8Bs4PjhDpR0paQPDXntd5L+IW1/XdKy9Hv8XtIeFcT/GuCOiLg6xf5kRPw4Ipam\nc0rSKenzfVDSRZI2y13/9ZJ+lUo/d0uamV5/saTzJN2fSpz/NuR7sVDSVyU9LOl2SYfnzrmjpF9K\nelzSfGDKkN/5R8pKro+m416Z2zcnfaeukPQk8K/pe5YvsU2XdPMon8nBwNbAx4B3Slo/996d0jUf\nTd/3/8l9TsN+/pL+XtKi9Prdkj6fO9/PJH14yO/3e0lHjfF3m7giwo8GPoA7gEPT9nbA74Gv5/af\nC3yxwnOtIssBbwq8FPgjcELadzzwPPAhshvgBsDXgcvS8RsBlwOnpuMPB/4GvBKYDFyQzv/yXFz/\nnrb3Bx7N/R5/B+yatq8G3peLcUNgKdnNdh2ym94DwO5p//+kRwewB3APcM0Iv+8OKaZ1UowrgENG\n+XwGj78nxXAOsEVu/6PAfrnn+wKPj3Cu9wALc89fCTwCrA9MA35LVtIA2BXYuoK/347AcuBrQCew\n0ZD9HwN+lT7f9YHvABekfS8DHgfeAawLbA7slfadB8xLn/3L0vfifbnvxXPACYCADwL35q75a7KE\ncH2ym/HjwHm5/cen866fvk+LcvvmpM90anr+IuAPwOG5Y+aRlXhG+ky+n74P6wEPAtNz+y4EPpW2\nJwGvS9sjfv7AIcAeaXtPsu/4Uen5McB1ufPvla65XtH3iaIehQcw0R/AncAT6R9rFbBg8Iub9s9J\nN4VH0uP+Uc61CujOPf9n4Kq0fTxwV26fgCdJN/T02lTg9rR9DikxSM93ZuQE4LvAGSPEdDUpEUrP\n38GQG3p6/+fIblzPAbvk9n0JuHaEc+/AmgRg27S9y3DHpuM3BPZJx78E+BHQl9u/Ysi1dwZWjXCu\njdPn99JcnN9L228iu8keAKxT5ffhAOAi4P70dz8X2DDtuw14U+7YbdLntS5ZteGlw5xvXeBZYLfc\na7OAq3Pfiz/n9k1On+NLgO3JMg0duf3nAz8YIfZN03s3zn135ww55pPAD9P25sBTwFYjnG8y8Bhw\nZHr+HeCy3P656buz7ZD3vbHSzx/4BvC1tL0B8DBZ9SFkCd+Z9fpfb8WHq4AaL8hyIJuQ5fp2B7Yc\nsv+rEbFZerxkjPMtzW3fTZZbHG7flmT/YDemKoNHgCtZU8TfZphzjWQ7YMko+/PtAC8DDhi8Zrru\nscBW6drrVXHdvEfIbj7bjBhExFORVQmtioj7gQ8D3VrT6P4ksEnuLS9Orw13rieAnwHvSi+9k+zm\nSET8AjiTrP5+maTvStq4kl8iIq6PiHekv/PBwBuAwYbqHYB5uc/tNrJEayuyv8Htw5xyClnu/K7c\na3eTJZiD/pa7/tNpcyOy784jEbE8d+zq80haV9JXUpXUY2Sl2cFrQvZ3z/8tIfuM3ippMvB2sszA\nsmE/DDg6/X5XpucXAG+WNHj+T5JlZG6QdKuk96bf4WpG+PwlHSDp6lQd9ijwT8AW6X3PkCW+70lV\nZO8E2rrHkxOAJoqIa8hyTbOH7FIVp9l+yPa9+Uvkth8ky2G+Mpe4bJoSIoD7hjnXSJYCO42wb2gj\n8N3AL3PX3CyyxtgPpZhWVHHdNRfJbly/Bt5WyfFDDH7P/8Da7S97AbeO8r4LgXcpa3zeIN14BuP5\nVkS8lqxqaBfgE9UGFRG/JasiGWw/uJus+iT/2U2OrL1iKcO3lzxIlovfIffa9mTVYGO5D9gs3awH\nvYw1f9NjgSPJqv4GG61hlO9rRNxL9neaDryb0W+wM8kSorsl3QdcTJaYHZvOtSwiZkXEtmQ38v9W\n6qY8yud/AVm153YRsSlZqSJ/n5sL/CNwGPB0RFw/SnwTnhOA5vsG0CXp1el5NTd/gI9L2lTSS4GP\nkuVoXiAiVgFnA9+QtCWApG0ldadDLgaOl7R7ugF8fsgplIvt+8B7Jb1J0jrpPLumfctY+8b0U2AX\nSe+WtH567Cdpt4hYCfwY6JXUkRoUZ1J5T6JPppg/LmmL9DvtJenCtL2/pF1TjFsA/0lWFfJEev95\nZA2VfydpW+BfyRLkkVxBdkP8Alk9Nek6r005zfWBp4FnqKAnj6SDJL0/9/fYDXgrcF065DvAqZK2\nT/u3lHRk2nc+cJikYyStp6wxfa/0mV4MfEnSRpJeBpwE/HCseCLiLrK69C+kv9PrgSNyh2xEVr30\ncCpFnTr0Vxrh1OcBJwOvIvt7D/dZbEtWlfb3ZAnx4OM0Um+g9Ltul97yKNn3ZNUYn/9GZKWa55R1\nKz2W3PcrIq4jK0nOTnG2NScATRYRD5J98T47+BLVdaX8X+BGYBHZzfb7o5znZOAvwHWpCL+ALLdE\nRPSRJUa/AP5E1ricf//q80XEb4D3kjUCPgoMsCbn/k3gbcp6mHwjIp4EusmK1/eS5TK/TNaIB1m1\nzEZk1RLnpMdo8v+8vya7abwJWCLpIbI64p+lQ15OVp3wOHALWQnoXbn3fxf4Sdr3e+AnEXHWiBeO\neI7sBnYoWc5y0CbAWWT1yXeS5cK/CiDp05KuGOGUj5LlqG+R9ESK9cfA6Wn/N8ka6udLepwsJ71/\nimUp8Bagh6zL5CKyrqwAHyGra78duJYssTh38Nfghd+L/PNjyerSHyZrp5mb23ceWZXQvWQlpV8z\nwndkiHlk3495qdplOO8ha1C+KiLuT49lwLeAPVOvnteSfXefIPvefzQi7mSUzx/4F+Df0+f3WYbP\nIJ1H1kA8ZiI50SmimntPFSeWziFL3e+PiD3Ta8cAvcBuZL0xbmrIxScoSauAnSJiuLpgs9KQ9Bdg\nVmovKRVJxwHvj4g3FB1L0RpZAjiXFw7QuYWs4eeaBl7XzAokaQZZ76oy3vwnk5USRiz5tZOGJQAR\ncS1Zz438a4sj4k+NumYbaExxzaxOJA2Q9c750BiHNp2kaWTdb+9j7Sq9trVe0QFY5SKiWdM7mNUk\nIjqLjmEkEdFP1v5kiRuBzczaVClLAJJc1WFmVoOIqLhreZElgFGDLHqI9NDH5z//+cJjaIWYyhqX\nY3JMEy2uJUuW0NnZydSpU1m8eDER1eebG5YApME5vwJ2lbRU0vsk/YOkpcCBwM8kXTn6WczMLG/V\nqlX813/9FwcccABHHHEE1157LbvuuuvYbxxGw6qAIuJdI+y6rFHXNDObyG6//XZOOOEEnn32WRYu\nXFjzjX+QG4Er1NnZWXQIL1DGmKCccTmmyjimyjUzrnrm+vMaNhJ4PCRFGeMyM2u2fK7/3HPPHfXG\nL4lokUZgMzMbQaNy/Xml7AZqZtbO6l3XPxKXAMxKpL+/n+7uGXR3z6C/v7/ocKzJmpHrz3MJwKwk\n+vv7OfromSxffhoACxfOZN68uUybNq3gyKwZmpXrz3MJwKwkzjjjrHTznwlkCcEZZ3jSyomu2bn+\nPJcAzMwKUkSuP88lALOS6OmZRUfHyWSLcs2lo+NkenpmFR2WNUCRuf48jwMwK5H+/v7V1T49PbNc\n/z8BVdOvv1rVjgNwAmBm1gSrVq3i29/+Nr29vZxyyimceOKJrLtufZf4qDYBcBuAmVmDFV3XPxK3\nAZiZNUhZ6vpH4hKAmVkDlDXXn+cSgJlZHZU915/nEoCZWZ20Qq4/r5Ergp0jaZmkW3KvbS5pgaQ/\nSZovadNGXd/MrFlaKdef17BuoJIOBp4EzouIPdNrpwMPRsTpkk4GNouIU4Z5r7uBmllLaGS//mqV\nZj2AiLgWeGTIy0eSDXMk/fyHRl3fzKyRWjXXn9fsNoCtImJZ2l4GbNXk65uZjVur1fWPpLBG4IgI\nSSPW8/T29q7e7uzsLO26oGbWPpoxmrcaAwMDDAwM1Pz+hk4FIWkH4Ce5NoDFQGdE/E3SNsDVEbHb\nMO9zG4CZlUqZ6vpHUpo2gBFcTjbZOennZU2+vplZVSZCXf9IGlYFJOlC4BBgiqSlwOeArwAXSzoB\nuBN4e6Oub2Y2XhOlrn8kng3UzGyIstX1V8qzgZqZjcNEz/XneS4gMzMmdl3/SFwCMLO21065/jyX\nAMysbbVjrj/PJQCzggyu//vgg8uA9ZgyZQuvA9xE7Zrrz3MJwKwA/f39HH30TBYs2JFFi/7IokXv\nZcGCIzn66Jn09/cXHd6E1u65/jyXAMwKcMYZZ7F8+WlkYyNnMzg+cvnybJ9LAY3hXP/aXAIwswnP\nuf7huQRgVoCenlksXDiT5cvfDXx89esdHSfT0zN35Dda1ZzrH5lHApsVxI3AjdWqo3nHo9qRwE4A\nzGzCaYWZOxuh7LOBmpk1jOv6q+M2ADObEFzXXz2XAMxaXH9/P93dM+juntGWYwic66+dSwBmLWxw\nQFk2pgAWLpzJvHlz26Yh2bn+8XEJwKyFrRlQNhPIEoIzzjir6LAazrn++iikBCDpY8D7AQFnR8Q3\ni4jDzFqPc/310/QSgKRXkd389wP2Ao6Q9Ipmx2FWpMF6+332eT377NNZc/19T88sOjpOBuYCH2ed\ndXp48MGH1jrXRGkjcK6/ASKiqQ/gbcD3cs8/A3xiyDFhNlH19fVFR8dWAT0BmwQcGHBgTJq0afT1\n9dV0vr33PijWWWezgDkBc6KjY6vo6+vLXWvt11vNkiVLorOzM6ZOnRqLFy8uOpzSSvfOyu/H1Rxc\njwewG/BHYHNgMvBr4JtDjmnQx2NWvK6u6emGfFDAlNU3Z5gSe+990DjPGekxJ7q6po/4eqtYuXJl\nnHnmmTFlypSYPXt2rFixouiQSq3aBKDpbQARsVjSacB84ClgEbBq6HG9vb2rtzs7O+ns7GxShGbN\n8jfyM4EC3HXXFwuLpmxc1z+2gYEBBgYGaj9BNalFIx7AqcAHh7xW/6TRrCTWVMu86gW58733PmSc\n52z9KiDn+mtHlSWAQuYCkvSSiLhf0vZAP3BARDye2x9FxGXWLP39/XzqU1/k5pv/QMQ3AJg06RNc\nfvkPau7DPzi5HLDWpHIjvV5G7TqHT720xGRwkq4BtgCeB06KiKuH7HcCYHVXxhthGWMqQjvO3NkI\nLZEAjMUJgNXb0BGzHR0nt/SI2YmUcDjXXz+eDdRsGBNpxOya9YSPbOl1hN2vv3ieC8isxaydmLXm\nOsLu4VMOLgFYW1h7xOzctPTirKLDajvO9ZeLSwDWFqZNm8a8eXNz9eatW/+/Zj3h7HmrrCPsXH/5\nuBHYrABr1gN+CFjBlClbVdWYm28EPuSQffjlL28Cytkg7B4+zeNeQGYlN7RHEnwcmElHxw+r7plU\n9t5N7uHTXO4FZFZyQ3skZdNB3FFTz6Sy9m5yXX9rcBuAmdWV6/pbh0sAZk02tEdSVgW0Y009k8rU\nu8m5/tbjNgCzAoy3EXi4c0FxjcCu6y8HNwKb1aAMN9FW5B4+5VJtAuA2AGt7Q3vSLFw4s+E9aSZC\nguO6/tbnNgBre83uSdPqc/m4rn/icAnArMlaeS4f5/onFpcArO2VqSdNWTnXPzG5BGBtr9nzBLXa\nXD7O9U9cY/YCknQjcA5wQUQ8UpeLSicBJwAB3AK8NyKeze13LyCb0FqhEdg9fFpP3buBStoZeC/w\nduC3wLnA/Frv0JK2Ba4Fdo+IZyVdBFwREXNzxzgBMCuQ+/W3prrPBRQRf46ITwO7ABeQlQbulvQF\nSZvXGOd6wGRJ6wGTgXtrPI9Z0/T399PdPYPu7hkt1WunGq7rby8VtQFI2ousFPBm4FKyhOD1wC+A\n11RzwYi4V9IZwN3AcqA/Iq6q5hxmzVbEWIFmc11/+xkzAUhtAI8B3wNOiYhn0q7rJB1U7QUlbQYc\nCeyQzvsjSf8YEefnj+vt7V293dnZSWdnZ7WXMqubVu26WUlbg+v6W9fAwAADAwM1v7+SEsDbI2LJ\ncDsi4ugarnkYcEdEPAQg6cfA64AREwAzq14lpRbn+lvb0MzxF77whareP2YCEBFLJB0BvBLoIOu5\nQ0T8e1VXWuMu4EBJHcAzZAnCDTWey6wpWq3rJoxeanGu36CyKqDvkt343wScTdYb6PpaLxgRN0i6\nBLgJWJF+Fr+ChdkoJtKaws7126BKRgK/LiKOAx6OiC8ABwLj+sZERG9E7B4Re0bEzIh4fjznM7MX\nGjrCeYMNPsnOO2/tHj62WiXjAG6IiP0lXQfMAB4Cbo2InRoWlMcBWMmUfe3dkQw2Aj/99FM88cR9\nbLjhhu7XP4E1Yk3gn6aeO18FbgTuBC6sLTyz1lTWtXfH0tXVxVFHvYk//vFGjjvuOOf6bS2VNAIP\nNvZeKumnwAYR8VhjwzIrvwcfXEZ39wygsukc1qwCtgxYjylTtqhqGohqp49wXb+NKSJGfQDHAJuk\n7c8CPwb2Get943lkYZmVR19fX3R0bBUwJ2BOTJq0aUyatOXq5x0dW0VfX18F7+8JmFLx+0a6/mjv\nW7lyZZx55pkxZcqUmD17dqxYsaLm39taS7p3Vn6vHfMAuCX9fD0wABwB3FDNRap9OAGwMurr64uu\nrunR1TU99t77kHQzjvSYE11d00d8b1fX9HT89Kre98L3j/6+JUuWRGdnZ0ydOjUWL148rt/XWk+1\nCUAlbQAr088jgLMj4qfA+nUpfpi1kGnTpjF//qXMn38pU6ZsUXQ4a/EcPlaLSkYC3yvpLKAL+Iqk\nDfBCMtbmqh0Ytub4dwMfX/16pQPKRrue6/qtVpV0A90QOBz4fUT8WdI2wJ4RMb9hQbkbqJXQ0EZY\noKpG2Xo3And1dXk0r62lEesBvAK4NyKekfRG4NXA3Ih4dHyhjnpNJwBWKmUbB+D5+m04jRgH8GNg\nhaSdgO8C25FNB23WNsoyDsB1/VZPlbQBrIqIFZKmA9+KiG9JWtTowMxsba7rt3qrpATwnKRjgeOA\nn6bX3AvISmnoql31WsVr6Lw6WSPsrLrFPRrn+q1RKmkD2AP4J+DXEXGhpB3J1gg4rWFBuQ3AajC0\nnn7SpBOB9Xnuua8C46+3L2Ihd9f1WzXq3gg8zAW2B94ZEadXG1wV13ACYFXr7p7BggVHMjj/PUwF\nPph7PpeursuZP//SQuKrhufrt1pUmwBUuibwlmTrALwL+DtgXm3hmdlYXNdvzTJiG4CkTSQdL6kf\n+A3wCmDHiHh5RPTUekFJu0palHs8JumjtZ7PbNDQevpJkxYzadInKKLevhau67dmG7EKSNJysqUa\nPwtcGxEh6Y6I2LFuF5fWAe4F9o+IpbnXXQVkNRnvYK1mysd67LFvZe7cua7rt3GpWxuApBPJqnw2\nJJv//yJgQZ0TgG7gcxHx+iGvOwGwCW1Ng/WXgV8A5/OBD7yfb3/7267rt5rVbSBYRHwjIg4Ajkov\nzQO2kXSypF3GGeegd+JBZdaGsoFlPcB5wBLgVO688yHf/K2pxhwHEBFLIuJLEbEnsB/wYuDK8V5Y\n0iTgrcCPxnsus1ayatUqli5dAnyRbJLda4Ftig3K2lJFvYAGRcQtwC3Ap+tw7TcDN0bEA8Pt7O3t\nXb3d2dlJZ2dnHS5pVqzBHj7Sc7zoRZN49tkpwA8rnhXULG9gYICBgYGa31/1OIB6kfQ/wJUR8YJv\nvdsAbKIZrl//VVddVdoGamtNDR8IVg9pium7yLqVPjHMficANmF4NK81S91nA5V0ZOquWTcR8VRE\nTBnu5m82Ubhfv5VdJTf2dwB/kXS6pN0aHZBZ2dQyodztt9/OoYceyvnnn8/ChQvp6ekZsYdPvSas\nM6taJQsHk/X8+SBwHfBrYBawcTWLD1fzwIvCW0n09fVFR8dWaUH2OdHRsVX09fWNePzKlSvjzDPP\njClTpsTs2bNjxYoVdT2/2WioclH4itsAJE0B3gOcCNwG7Az8Z0T8Z70TJbcBWFm8cIK5kSeUq6Wu\nv5rzm42lEW0AR0maBwyQrQOwX0S8mWxpyH+tNVCzshhvFYzr+q1ljVVEIJtJ6w0j7DusmuJGpQ9c\nBWRNMlYVzFj7lyxZEp2dnTF16tRYvHhx3a9vVg2qrAIa60a8HjBQzQnr8XACYM3S1TU93XwjPeZE\nV9f0tY7p6+uLrq7p0dU1ffXNudq6/tEMd36zWlSbAIw6EjiytYBXSto0Ih5tXDnErLymTZu21iCt\nes/XP/T8Zs1SSTfQp4BbJH1f0rfSo+4Nv2ZFqGatX9f120RTyZrAxw/zcsQwUzjUi3sBWTNVstav\nR/NaK2iJqSDG4gTAyqLotXmLWIjeWlfd1wROc/+fCrwS6EgvR0S8vLYQzVpD0Wvzrlk05jQAFi6c\nybx5c50IWN1U0gZwLvAdYAXQSVZZen4DYzIrVL6uf+edd2bDDbfmIx/5dNOnacgWjTmNbJBYlhAM\nlgbM6qGS9QA6IuIqZfUydwG9km4iWyvYbELJ5/q//OUv89GPfsY5cJuwKikBPCNpXbIJ4T4saTrZ\nOsFmE8ZwPXwuvvjKQnPg1fRQMqtFJSWAE4HJwEfJ1rDbhDUTl5i1vKLr+kcybdo05s2bm2sEdunD\n6su9gKxtjdXDZ2gjbEfHya4CslKrWzdQST/JPQ1AuW0i4shxBLkp8D1gj3S+90XEdbn9TgCsoSrt\n119rN0x337Qi1DMB6EybRwNbAz8kSwTeBSyLiBPHEeRc4JcRcY6k9YANI+Kx3H4nANYQzejX75KD\nFaXuA8Ek3RgR+471WhUBvhhYNNo4AicA1gjNGs3rOf6tKHVfDwCYLOkVuQu8nKxRuFY7Ag9IOlfS\nTZLOljSe85mNynP4mA2vkl5AJwFXS7ojPd+BbEnI8VxzH+DDEfEbSd8ATgE+N45zmg2riB4+PT2z\nWLhwJsuXZ8+z7psNmzrLrGZjJgAR0Zemg9iNrMF2cUQ8O45r3gPcExG/Sc8vIUsA1tLb27t6u7Oz\nk87OznFc0tpNkXP4uPumNcvAwAADAwM1v7+ibqCSXkdWdbMea3oBnVfzRaVrgPdHxJ8k9ZKNNj45\nt99tAFYzz9xp7aoRawL/EJgNHAS8FtgvPcbjI8D5kn5HtrbwqeM8n5nr+s2qVEkbwL7AK+uZJY+I\n3zH+RMRstbKO5jUrs0p6Ad0KbNPoQMxq4Vy/We0qKQFsCdwm6QZgsPE3xjMS2KwenOs3G59KEoDe\n9HOwCki5bbOmK3qVLrOJopJuoAOStiarsw/ghoi4v+GRmQ3DuX6z+qmkF9DbgeuBY4C3AzdIOqbR\ngZnlua7frP4qqQL6DLDfYK5f0pbAz4EfNTIws0HO9Zs1RiW9gAQ8kHv+EGumhrY20N/fT3f3DLq7\nZzR1XVzn+s0aq5ISQB/QL+kCshv/O4ArGxqVlcbQqY2btS6uc/1mjTdmCSAiPgF8l2zE7p7AdyPi\nk40OzMrhjDPOauq6uM71mzXPmCUASTsCV0TEpel5h6QdIuLORgdn7cW5frPmqqQN4BJgZe75qvSa\ntYGenll0dJwMzAXmpqmNxzMb+As51z+6otpgbOKrZEWwmyPiNUNe+11E7NWwoDwbaKk0cn1bz9w5\nOi8vadWodjZQImLUB3AVcFTu+VHAz8d633geWVg2UfT19UVX1/To6poefX19ERGxcuXKOPPMM2PK\nlCkxe/bsWLFiRcFRllNX1/SAOQGRHnOiq2t60WFZSaV7Z8X32kp6AX2QbOrmM9Pze4D3VJ4mWTsb\nrhfRf//3V5g7d67r+s0KVtGCMACSNkrHP9HYkFwFNJGsvUD6KmAm669/MV/+8qmew6cCrgKyajRi\nQZitJX0fuCQinpD0SkknjCtKa0O3A4cC17HffofQ09Pjm38FBpeX7Oq6nK6uy33zt7qqpBG4DzgX\n+LeIeLWk9YFFEfGqmi8q3Qk8Tta76PmI2H/IfpcAJogrr7ySo456B88/D/AWNtjgai677DzfxMwa\noNoSQCVtAFMi4iJJpwBExPOSVtQcYSaAzoh4eJznsRK7/fbbOf3009lpp+3YbLPt2HDD5+np8c3f\nrCwqSQCelLTF4BNJBwKP1eHank9ogvJ8/WatoZIEoAf4CfBySb8iWyHsbeO8bgBXSVpJNrXE2eM8\nn5WER/OatY5KFoS5UdIhwOB/8h8j4vlxXvegiLgvTS29QNLiiLg2f0Bvb+/q7c7OTjo7O8d5SWsk\n5/rNmm9gYICBgYGa3z9iI7Ck/YGlEXFfej4TmAHcCfTWq/5e0ueBJyPijNxrbgRuIWON5m3kSGIz\nW6Oe3UC/S1oEXtIbgK+QTQjzOFDzdJCSJkvaOG1vCHQDt9R6PitOJXP4DPZjX7DgSBYsOJKjj57p\n+WzMSmLCwaCYAAAOFklEQVS0KqB1crn8d5DV1V8KXCrpd+O45lbAPEmD1z8/IuaP43xWgErr+tee\nThqWL89ecynArHijlQDWTX3+AQ4Drs7tq6TxeFgRcUdEvCY9XhURX671XNZ8nrnTbOIY7UZ+IfBL\nSQ8CTwPXAkjaGXi0CbFZydTSw6enZxYLF85k+fLseTad9NwGR2pmlRh1JLCkqcDWwPyIeCq9tguw\nUUTc1LCg3AhcKuPt4eNGYLPmqLYRuOLJ4JrJCUB5eL5+s9ZR98ngrD25rt9s4qu5MdcmLo/mNWsP\nLgHYas71m7UXlwAMcK7frB25BFBy/f39dHfPoLt7RkNG0DYj19/o38HMauNeQCXW6OUAm9HDx0sa\nmjWPewFNIGtPo5DdRAf7049HM+v6G/U7mNn4uQ2gzbiu38wGuQRQYj09s+joOJlsEta5aRqFWTWd\nq6gePvX8HcysvtwGUHL1mEah6NG8ngrCrDk8FYSt5lW6zNpLtQmA2wAmKNf1F88lHys7twFMMB7N\nWw5eCc1aQWElAEnrAr8F7omItxYVx0TiXH95eCU0awVFlgA+BtwGuLJ/nJzrN7NaFFICkLQd8Bbg\nS8C/FhHDROFcfzl5JTRrBUWVAL4OfAJYVdD1W55z/eU2bdo05s2bS1fX5XR1Xe7pL6yUml4CkHQE\ncH9ELJLUOdJxvb29q7c7Ozvp7Bzx0LbjXH9rmDZtmm/61lADAwMMDAzU/P6mjwOQdCrwHmAFsAGw\nCXBpRByXO8bjAIbhfv1mNpqWGggm6RDg40N7ATkBeKGiR/OaWfm14mygvtOPwnX9rcPrHlir8VQQ\nJeZcf+vwugdWBq1YArAhnOtvPV73wFqR5wIqmUp7+HieGTMbL5cASqKaXL/nmSkfr3tgrchtACVQ\nbV1/d/cMFiw4ksF5ZiAbcDR//qUNj9VG5lKZFc3TQbcQ9+ufWDzwy1qNE4CCjGc0r+eZMbN6cBtA\nk9Wjh4/nmTGzenAbQBO5X7+ZNZLHAZSQ+/WbWRm5DaDBPHOnmZWVSwAN4ly/mZWdSwAN4Fy/mbUC\nlwDqyLl+M2slLgHUSSW5fo8UNbMycQlgnCrN9Xv+HjMrG48DGIdq+vW/cP6ej7P55pex7757uTRg\nZnVR+nEAkjaQdL2kmyXdKqm32TGM1/jr+vuBuTz88GddGjCzwjS9DSAinpH0xoh4WtJ6wEJJV0bE\n9c2OpRa19vBZe/6e7wCzGSwNLF+eLSjiUoCZNVMhbQAR8XTanASsD6wqIo5qjDfXn5+/Z/PNH2hg\npGZmlSmkF5CkdYCbgFcAZ0bEb4qIo1L16tc/OF3wmvVjs9c9m6eZFaGQBCAiVgGvkfRiYJ6kPSLi\nD/ljent7V293dnbS2dnZ1BihcfP1D5YG1nQJ9WyeZla9gYEBBgYGan5/4b2AJH0WeDoizsi9Vngv\nIM/caWatphV6AU2RtGna7gC6gP/X7DhG4tG8ZtYuiqgC2gaYK2ldsgToooi4ooA4XsBz+JhZO2l6\nCSAibomIfSJir4jYMyL+o9kxDNXIXH9/fz/d3TPo7p7hvv5mViptPxdQI3P9a3r7nAbAwoUzvXyj\nmZVG284F1Iy6/jPOOCvd/GcCWUIw2PPHzKxobVkCcF2/mVmblQCa3cOnp2cWHR0nA3OBuWnA16yG\nXc/MrBqFjwMYTiPGARTVr99rAJhZs1Q7DmDCJwCNGs1rZlY21SYAE7oNwHX9ZmYjm5BtAB7Na2Y2\ntglXAnCu38ysMhOmBOBcv5lZdSZECcC5fjOz6rV0CcC5fjOz2rVsCcC5fjOz8Wm5EoBz/WZm9dFS\nJQDn+s3M6qeIFcFeKulqSX+QdKukj471Huf6zczqr4gqoOeBkyJiD+BA4EOSdh/p4Ntvv51DDz2U\n888/n4ULF9LT01PIVA7jWXi5UcoYE5QzLsdUGcdUubLGVY0iVgT7W0TcnLafJFsP+O+GHle2XH8Z\n/9hljAnKGZdjqoxjqlxZ46pGoW0AknYA9gauH7rv0EMPdV2/mVkDFdYLSNJGwCXAx1JJYC1lyPWb\nmU1khUwHLWl94KfAlRHxjWH2l2+OajOzFlDq9QAkiWyJrIci4qSmXtzMzFYrIgF4PXAN8Htg8OKf\nioi+pgZiZtbmSrkimJmZNV6ppoKoZZBYE2LaQNL1km5OMfUWHdMgSetKWiTpJ0XHAiDpTkm/TzHd\nUHQ8AJI2lXSJpP8n6TZJB5Ygpl3TZzT4eKwk3/WT0nf8FkkXSHpRCWL6WIrnVkkfKyiGcyQtk3RL\n7rXNJS2Q9CdJ8yVtWpK4jkn3z5WS9hnrHKVKAKhykFgzRMQzwBsj4jXAa4DDJR1QZEw5HwNuY01V\nWtEC6IyIvSNi/6KDSb4JXBERuwOvJht3UqiI+GP6jPYG9gWeBuYVGZOkbYGPAPtGxJ7AusA7C47p\nVcD7gf2AvYAjJL2igFDOBQ4f8topwIKI2AX4eXrebMPFdQtwNFk1+5hKlQBUOkis2SLi6bQ5CVgf\nWFVgOABI2g54C/A9oOJW/yYoTSySXgwcHBHnAETEioh4rOCwhjoMWBIRS4sOhGxc0GRJ6wGTgXsL\njmc34PqIeCYiVgK/BKY3O4iIuBZ4ZMjLR5J1ZiH9/IemBsXwcUXE4oj4U6XnKFUCkDfaILFmk7SO\npJuBZcD8iPhN0TEBXwc+QQkSo5wArpL0W0kfKDoYYEfgAUnnSrpJ0tmSJhcd1BDvBC4oOoiIuBc4\nA7gb+CvwaERcVWxU3AocnKpbJgN/D2xXcEyDtoqIZWl7GbBVkcHUqpQJwFiDxJotIlalKqDtgAMk\n7VFkPJKOAO6PiEWUKMcNHJSqNd5MVn13cMHxrAfsA/x3ROwDPEUxRfVhSZoEvBX4UQli2YwsV7sD\nWal7I0n/WGRMEbEYOA2YD1wJLKJcGR4AIutJU5Zq2KqULgFIg8QuBX4YEZcVHU9eqj64mhfWuzXb\n64AjJd0BXAi8SdJ5BcdERNyXfj5AVqdddDvAPcA9uRLbJWQJQlm8GbgxfV5FOwy4IyIeiogVwI/J\nvmeFiohzIuK1EXEI8Cjwx6JjSpZJ2hpA0jbA/QXHU5NSJQBpkNj3gduGGyFcBElTBlv4JXUAXRTc\nkBgRn46Il0bEjmRVCL+IiOOKjEnSZEkbp+0NgW6yBqnCRMTfgKWSdkkvHQb8ocCQhnoXWQJeBncB\nB0rqSP+Hh5F1MCiUpJekn9uTNW4WXl2WXA7MTNszgVJlVpMxawfKtiDMQcC7gd9LWpReK3qQ2DbA\nXEnrkiWYF0XEFQXGM5wyFD+3AuZl9w7WA86PiPnFhgRkPVvOT9UtS4D3FhwPsDqRPAwoQ1sJEXGD\npEuAm4AV6edZxUYFwCWStiDrIfgvEfF4swOQdCFwCDBF0lLgc8BXgIslnQDcCby9BHF9HngY+BYw\nBfiZpEUR8eYRz+GBYGZm7alUVUBmZtY8TgDMzNqUEwAzszblBMDMrE05ATAza1NOAMzM2pQTACuV\nNI3tojQF8MVp8F2t55ojaUbaPnu0mWUlHSJpag3XuFPS5jXGtL6kr6QphW+U9CtJh6d9A5IW56aM\n3jK9/iJJF0n6s6TrJL2s2pjNBjkBsLJ5Ok2VvCfwHPDB/M40U2WlVs/REhEfiIjRRnC/kdqmPqh2\nIE1+3pgvkg2g2yMi9iWbUXLj3HHHDk4bnZsu4gSy5VR3JpsQ8LQaYjYDnABYuV0L7JRy59dK+l/g\n1jQ761cl3SDpd5JmQTaViKQzU855AfCSwROlHPW+afvwlOO+OS3q8TLgn4CTUm77IElbKltI5ob0\neF167xbKFgC5VdLZjDDcXtKTkr6WjrtK0pQh+yeTzXX/kYh4HiAi7o+I/MRww507Pw3xpcCh6Xzb\nSLomV3p6fVWftLUlJwBWSimn/xaytaMhmxr8oxGxG9mN89G06Mz+wAfS9OFHA7sAuwPHsXaOPoBI\nVSlnAdPTDK/HRMRdwHeAr6Xc9v+RLSTz9XSNt5GtuwDZcPtrIuJVZBPebT/CrzAZ+E067pfpfat/\nPWAn4O4xZrudm27on8m9ti2wFLL1DYDH0lQJxwJ9aTbWVwM3j3JeM6B8cwGZdeTmgboGOIdsjqgb\n0o0asonm9pT0tvR8E2Bn4GDggjQ9732SfjHk3CJbae6awXNFxKND9g86DNg9zW0EsHGav+dgsoSG\niLhC0tCFQgatAi5K2z8km10zb6yqo3+MiL8qmxr9UknviYgfjHBsADcA56TZdC+LiN+NcX4zJwBW\nOstTLna1dBN+ashxH46IBUOOewtjz4BYaZ29gAMi4rlhYql2DQYNuW4AfwG2l7RxRDzxgiAj/pp+\nPinpArKSzg/IVunaHvhrKiW9OCIeBq5Vtv7CEcAcSV8bJcEwA1wFZK2pH/iXwQZhSbukOvVrgHek\nNoJtyBp28wK4DnhDqjIi14PnCdY0wEK2CMnqhdol7ZU2ryGrbkHSm4HNRohxHeCYtH0sWXvGmkAi\nlpNNff7NlGsntTu8TdK6g20Gad9bWTO1dn4a4reRrUc7OF3yAxHxPbLqqrUSUbPhOAGwshkuhz50\nxaXvkc1Vf5OkW4BvA+tGxDzgz2nfXOBXLzhRxIPALODHypb5HJyP/yfA0YONwGQ3/9emRuY/kDUS\nA3yBLAG5lawq6C6G9xSwf4qvE/j3YY75DPAAcFs67ifAY8CLgD5JvyNbBWspcHZ6z/eBLST9GTiR\nNSucdQI3S7qJbGrib44Ql9lqng7arAEkPRERG499pFlxXAIwawznrKz0XAIwM2tTLgGYmbUpJwBm\nZm3KCYCZWZtyAmBm1qacAJiZtSknAGZmber/AyHFAVkhJnhiAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f6d082b3a50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "task = \"pIC50\"\n",
    "rf_predicted_test = np.array(rf_test_df[task + \"_pred\"])\n",
    "rf_true_test = np.array(rf_test_df[task])\n",
    "plt.scatter(rf_predicted_test, rf_true_test)\n",
    "plt.xlabel('Predicted pIC50s')\n",
    "plt.ylabel('Secondary Assay')\n",
    "plt.title(r'RF predicted IC50 vs. Secondary Assay')\n",
    "plt.xlim([2, 11])\n",
    "plt.ylim([2, 11])\n",
    "plt.plot([2, 11], [2, 11], color='k')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEZCAYAAACervI0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYXGWZ/vHvnRC0wyJLFBBkkVUWURAwgtIK3UEGoySi\nqKNBcaIzioAtAzoqraKCEpcRRwWERA2MC8YfjJBOcGihRUDZDDJxCftiWAQUiGLo5/fHeTs5qfRS\n1V1Vp5b7c111peqcU+c8VV057/6+igjMzKz9TCo6ADMzK4YTADOzNuUEwMysTTkBMDNrU04AzMza\nlBMAM7M25QTAxiRpUNKL0/NvSPp4Ha55nKRran0dK4+kXknfLToOqy4nAFUi6S5JT0v6i6THJP1C\n0vskKXfM/HQzPSC3bRdJg7nX/ZJWSdout+1wSXfW79OMLCL+NSLOGOu49DmOr0UMknZM3+Ok3LYD\nJV2evvtHJV0v6biS4/+ae/xHyTnPkvRIepxZi7hHI+mNkm6R9ISkhyX9TNKO9Y5jFHUfMJQSnUFJ\nB9b72u3CCUD1BHBURGwKbA+cCZwKfLvkuD8DY91AnwI+UfUIAUmTa3HeYdTthiFpOvAz4Cpg54jY\nEvhX4IiSQzeNiE3S47O5978PeCPw0vR4Q9pWF5J2ARYAJ0fE84CdgK8Dz9YrhlrKJ9QVvEfAu4BH\n079WA04AaiAi/hoRlwFvBeZI2nNoF9l/9JdKes1Ibwf+E3jbULXLWFIu6QRJK1Lu8QtDJY9UlfIL\nSV+S9AhwuqQNJZ0t6W5Jf0rVOs/Nne8USQ9Iuk/Se0quNV/SZ3Kv8znXP0qaIemzwKuBc1Ju+z/T\nsXtIWppy6MslHZM7z5aSLk3nuR7YuZzPnnwRmB8RX4yIPwNExE0RcWzJcSP93ucAZ0fEAxHxAHA2\ncNxwB0q6QtIHSrbdKulN6fmXJa1Mn+M3kvYqI/6XAXdGxFUp9icj4scRcW86pySdlr7fRyR9X9Lm\nuesfIunaVPq5R9KctP15kr4j6aFUQv2Pkt/FgKQvSvqzpDskHZE7506Sfq6sRLsEmFbymX8o6UFJ\nj6fj9sztm59+U5dLehL4cPqd5UtssyTdMsp38mpga+BE4FhJU3Lv3SVd8/H0e//v3Pc07Pcv6Z8k\n3Zy23yPp9Nz5firpgyWf7zeS3jjG3635RYQfVXgAdwKvG2b73cD70vMLgc8AJwDXpG27AIO5468C\njgfmAd9N2w4nu0GMdO1BshzwZsCLgN8Bx6d9xwH/AD5AdgN8LvBl4Cfp+I2BS4HPpeOPAP4E7AlM\nBS5K539x7jN8Oj0/EHgcOCy9fiGwe+5zvCcX40bAvWQ320lkN72HgZek/f+dHh3AXsB9wNUjfN4d\nU0yTUoyrgUNH+X6Gjr8vxXABsGVu/+PAAbnX+wN/GeFc7wQGcq/3BB4DpgAzgF+TlTQAdge2LuO3\nsxOwCvgS0AlsXLL/RODa9P1OAb4JXJT27QD8hSyzMRnYAtg37fsOsCh99zuk38V7cr+LZ8h+awLe\nD9yfu+YvyRLCKWQ3478A38ntPy6dd0r6Pd2c2zc/fafT0+vnAL8Fjsgds4isxDPSd/Lt9HvYAHgE\nmJXbdzHw0fR8Q+BV6fmI3z9wKLBXer4P2W/8jen1McB1ufPvm665QdH3lVo/Cg+gVR6MnAD8Mvdj\nvRD4dPrR3k12sx0uAXgPWY7r8XSDKScB6M69/lfgyvT8OODu3D4BT5Ju6GnbdOCO9PwCUmKQXu/K\nyAnAt4B5I8R0FSkRSq/fSskNPb3/k2Q3rmeA3XL7PktKJIc5946sTQC2Tc93G+7YdPxGwH7p+BcA\nPwQW5/avLrn2rvm/Scm5Nknf34tycZ6fnr+O7CZ7EDCpwt/PQcD3gYfIEoMLgY3Svtvzvy1gm/R9\nTQY+ClwyzPkmA38H9shtmwtclftd/CG3b2r6Hl9AVoX5D6Ajt38hKUMyzLU2S+/dJL2eT1Yiyx/z\n78D30vMtyKo5txrhfFOBJ4CZ6fU3gZ/k9i9Iv51tS9732nK/f+ArwJfS8+eSVc3unF6fDZxT6T2g\nGR+uAqq97ch+XGtExDNkJYHPMEJdeUQ8ApxDlmCUU59+b+75PWS5xeH2PZ/sP9iNqcrgMeAK1hbx\ntxnmXCPZDlgxyv583DsABw1dM1337cBW6dobVHDdvMfIbj7bjBhExFORVQkNRsRDwAeBbkkbpUOe\nBDbNveV5adtw5/or8FPgbWnTsWQ3RyLif8n+Zl8HVkr6lqRNyvkQEXF9RLw1Il5AluN+DTDUUL0j\nsCj3vd1OlmhtRfY3uGOYU04jy53fndt2D1mCOeRPues/nZ5uTPbbeSwiVuWOXXMeSZMlnZmqpJ4g\ny/wMXROyv3v+bwnZd/QGSVOBt5BlBlYO+2XA0enzXZFeXwS8XtLQ+f+dLCNzg6TbJL07fYarGOH7\nl3SQpKtSddjjwPuALdP7/kaW+L4zVZEdC7RFjycnADWkrLfPC4GB/Ob073yynNPsUU7xRbJczf5l\nXG77kuf3517nb8SPkOUw94yIzdNjs8garwEeHOZcI7mXrAQznNJE6x7g57lrbh5ZY+wHUkyrK7ju\n2otkN65fAm8u5/gSQ7//35JVSQ3ZF7htlPddTNZGMx14brrxDMXztYh4BVnJbTfglEqDiohfk1WR\nDLUf3ENWfZL/7qZG1l5xL8O3lzxClovfMbdte7JqsLE8CGyebtZDdmDt3/TtwEyyqr+hRmtY+9se\n7jPdT/Z3mgX8M6PfYOeQJUT3SHoQ+AFZYvb2dK6VETE3IrYlu5H/l1J72Sjf/0Vk1Z7bRcRmZKWK\n/P1vAfAOstL20xFx/SjxtQwnANU11MC2qaSjyG4U342I3+b3A0TEauB0sp5Cw54nIp4gawsY7phS\nH5G0maQXAR8iy9GsJyIGgfOAr0h6fop3W0nd6ZAfAMdJekm6AZxecgrlPse3gXdLep2kSek8u6d9\nK1n3xvQ/wG6S/lnSlPQ4QNIeEfEs8GOgV1JHalCcQ/k9if49xfwRSVumz7SvpIvT8wMl7Z5i3JKs\nkf2qlJuHrK78w5JeKGlb4MNkCfRILie7IX6KrJ6adJ1XpJzmFOBp4G+U0ZNH0sGS3pv7e+wBvAG4\nLh3yTeBzkrZP+58vaWbatxA4XNIxkjZQ1pi+b/pOfwB8VtLGknYATga+N1Y8EXE3WV36p9Lf6RDg\nqNwhG5NVL/05laI+V/qRRjj1d8h+y3uT/b2H+y62JatK+yeyhHjocRapN1D6rEPdpB8n+50MjvH9\nb0xWqnlGWbfSt5P7fUXEdWQlybNTnO2h6DqoVnmQFYOfJmssexz4BVldvHLHrKk/T68FLAOezW0b\nrvF0JamOfoRrD5JVa6wgy/l9cei6ZDfS0rr355DVXa8gq2u9Hfhgbv+pZLnA+4B3k/0nWq8NIL1+\nE3Br+tx/ALrS9leS1cf+GfhK2rYbWULwUIrzSuClad804LIUz3VkVV+jNQI/S66eFziA7Mb8OFnX\nweuAf077jiWrJnkSeIDs5v6CknOeld73KHBmGX/v81MM++e2vS59F38la+D+LjA17fsYcPkI59qL\nrCH+T+m9dwKfBybnficnA8vT9/xH4Izc+w9Jn/cJstLCO9P2zVIMD6XtHx/jd5H/O+8EXJ3iWUKW\naH4n95v8SYrlTrKG8RF/I7nzd6QYLxzlez0N+NUw219Ilujslf5W96XY/gi8t4zvfzZwV4r5svzn\nyV3jP8j+L+1Y9P2kXo+hH0PVSbqALBV/KCL2SduOAXqBPch6XdxUk4u3GWUDyXaJiOHqgs0ahqQ/\nAnMjay9pKJLeRZaYjNRFu+XUsgroQtYfiLOMrIHn6hpe18wakKTZZL2rGvHmPxX4N+DcomOpp5ol\nABFxDVkPjfy25RHx+1pds43VphhnViWS+sl653xgjEPrTtIMsmqyB8kai9vGBkUHYBMXEfWa3sFs\nXCKis+gYRhIRfWSNxG3HvYDMzNpUQ5YAJLlKw8xsHCJixPEYpYosAYwaZNHdo0ofp59+euExNENM\njRqXY3JMrRbXihUr6OzsZPr06SxfvpyIyvPNNUsA0iCca4HdJd0r6T2S3iTpXrI+4j+VdMXoZzEz\ns7zBwUG+/vWvc9BBB3HUUUdxzTXXsPvuu4/9xmHUrAooIt42wq6f1OqaZmat7I477uD444/n73//\nOwMDA+O+8Q9xI3CZOjs7iw5hPY0YEzRmXI6pPI6pfPWMq5q5/ryajQSeCEnRiHGZmdVbPtd/4YUX\njnrjl0Q0SSOwmZmNoFa5/ryG7AZqZtbOql3XPxKXAMxqoK+vj+7u2XR3z6avr6/ocKxJ1CPXn+cS\ngFmV9fX1cfTRc1i16iwABgbmsGjRAmbMmFFwZNbI6pXrz3MJwKzK5s07N9385wBZQjBvXltNMmkV\nqHeuP88lADOzghSR689zCcCsynp65tLRcSrZMrML6Og4lZ6euUWHZQ2kyFx/nscBmNVAX1/fmmqf\nnp65rv+3NSrp11+pSscBOAEwM6uDwcFBvvGNb9Db28tpp53GSSedxOTJ1V3Ko9IEwG0AZmY1VnRd\n/0jcBmBmViONUtc/EpcAzMxqoFFz/XkuAZiZVVGj5/rzXAIwM6uSZsj159VyRbALJK2UtCy3bQtJ\nSyX9XtISSZvV6vpmZvXSTLn+vJp1A5X0auBJ4DsRsU/a9gXgkYj4gqRTgc0j4rRh3utuoGbWFGrZ\nr79SDbMeQERcAzxWsnkm2fBI0r9vqtX1zcxqqVlz/Xn1bgPYKiJWpucrga3qfH0zswlrtrr+kRTW\nCBwRIWnEep7e3t41zzs7Oxt2XVAzax/1GM1bif7+fvr7+8f9/ppOBSFpR+CyXBvAcqAzIv4kaRvg\nqojYY5j3uQ3AzBpKI9X1j6Rh2gBGcCnZJOmkf39S5+ubmVWkFer6R1KzKiBJFwOHAtMk3Qt8EjgT\n+IGk44G7gLfU6vpmZhPVKnX9I/FsoGZmJRqtrr9cng3UzGwCWj3Xn+e5gMzMaO26/pG4BGBmba+d\ncv15LgGYWdtqx1x/nksAZqy7hu+hh+7Hz39+E+D1fFtZu+b689wLyNpeX18fRx89h1WrzkpbPgT8\nC7APHR2nsmjRAicCLaRZe/iUw72AzCo0b9656eY/J7f1UuBsVq3K9jsBaA3O9a/LbQBm1vLava5/\nJC4BWNvr6ZnLwMAcVq0a2jJUBbSAjo5T6elZMPKbreE51z8ytwGY4UbgVtTKdf0jqbQNwAmAmbWc\nZpi5sxYafTZQM7OacV1/ZdwGYGYtwXX9lXMJwKyB9fX10d09m+7u2fT19RUdTkNyrn/8XAIwa1Cl\nA9QGBuZ4UFoJ5/onxiUAswa17gC1LCEY6qnU7pzrr45CSgCSTgTeCwg4LyK+WkQcZtZ8nOuvnrqX\nACTtTXbzPwDYFzhK0s71jsNsJNWodx/tHOWev6dnLh0dpwIfAaYzaVIPhx6637jiaQXO9ddARNT1\nAbwZOD/3+uPAKSXHhFkRzjjjjJg0acuAVwb0REfHVrF48eKKzrF48eLo6NgqYH7A/HXOMdq+kePZ\nvOzjW9WKFSuis7Mzpk+fHsuXLy86nIaV7p3l348rObgaD2AP4HfAFsBU4JfAV0uOqdHXYzayxYsX\nr3Ozha0CeqKra1ZF5+nqmpXeH+kxf805RttX6bnawbPPPhvnnHNOTJs2Lc4+++xYvXp10SE1tEoT\ngLq3AUTEcklnAUuAp4CbgcHS43p7e9c87+zspLOzs04RWruaN+9cBge/zLqzgn4TeGFBEbU31/WP\nrb+/n/7+/vGfoJLUohYP4HPA+0u2VT9pNBvDcLntSZO2LLQKqNLjW4Fz/eNHo1cBZTHygvTv9sD/\nAZuW7K/Jl2M2mtKb7aRJm8cZZ5wx7nN1dc2Krq5Z692wR9tX6blajev6J6bSBKCQyeAkXQ1sCfwD\nODkirirZH0XEZa0lP8NnubN6juc9NnHtOHNnLXg2UDPWH0Vbz6UdnYhUpl1n7qwFzwZqRnGjaIcS\nnqVLZ7J06UyOPnqO5/AZgfv1F89zAZlVUen6wl5TeHju4dMYXAKwlrR2FO0C1i7tOLfosNqec/2N\nxSUAa0kzZsxg0aIFubr4+tT/l64v7DWF13Kuv/G4EdhsFNXqSVS6DWibhmL38Kkf9wIyq5Jq9SQq\nPc+GG54ETOGZZ744ofM2A/fwqa9KE4DCRwIP98ADwawBVGsenvXP88qWn9/Ho3mLQaPPBWRmrc11\n/c3DCYDZCKrVoFt6ng03XA6cwjPPTOy8jcZ1/c3HbQBmo6jWqN5WbwR2XX9jcCOwGZ6OoV6c628s\nlSYArgKyllPa62ZgYE5Ne9m0a2Ljuv7m55HA1nLqOQ9QO87949G8rcMlALMJaLe5f5zrby0uAVjL\n8TxA1edcf2tyCcBaTj3nAWqHuX+c629dY/YCknQjcAFwUUQ8VpWLSicDxwMBLAPeHRF/z+13LyBr\nGq3aCOwePs2n6t1AJe0KvBt4C/Br4EJgyXjv0JK2Ba4BXhIRf5f0feDyiFiQO8YJgFmB3K+/OVV9\nRbCI+ENEfAzYDbiIrDRwj6RPSdpinHFuAEyVtAEwFbh/nOexNtfX10d392y6u2e3fO+benBdf3sp\nqw1A0r5kpYDXA5eQJQSHAP8LvKySC0bE/ZLmAfcAq4C+iLiyknOYQf37+7c61/W3nzETgNQG8ARw\nPnBaRPwt7bpO0sGVXlDS5sBMYMd03h9KekdELMwf19vbu+Z5Z2cnnZ2dlV7KWlwjdcFs5nYA1/U3\nr/7+fvr7+8f9/nJKAG+JiBXD7YiIo8dxzcOBOyPiUQBJPwZeBYyYAJg1smYuiTjX39xKM8ef+tSn\nKnr/mAlARKyQdBSwJ9BB1nOHiPh0RVda627glZI6gL+RJQg3jPNc1sYapQtmI5VEyuVcv0F5VUDf\nIrvxvw44j6w30PXjvWBE3CDpR8BNwOr0b23G6VtLK2rd32bnXL+tMdaKMcCy9O9v0r8bAwOVrDpT\n6QOvCGYNZvHixdHVNSu6umbF4sWL19vX0bFVWuVrfnR0bLXeMY3Aq3S1PmqwIlgqYPN06sP/KLB1\ntRMis0Y1Vh1/M5REnOu34ZQzEOyTwNfIqoC+njafFxGfqFlQHghmDaS7ezZLl85kqI4fFtDVdSlL\nllxSZFhlcV1/e6nFQLBPR8RjEXEJsAOwey1v/tY6mmWQVuVxLuPGG28d8fhG+dx33HEHhx12GAsX\nLmRgYICenh7f/G1dY9URAccAm6bnnwB+DOxXST1TpQ/cBtD0mqVevJw41z2mJ2DTEY9vhM/tuv72\nRYVtAJU0Ah8C9ANHATdUcpFKH04Aml9X16x0E4z0mB9dXbOKDms95cY51Ai8xRY7j3p80Z97xYoV\n0dnZGdOnT4/ly5fX7brWGCpNAMpZD+DZ9O9RZHX//wNMqUrxw6xJzJgxgyVLLmH//fctOpRheQ4f\nG5exUgjgp2T99O8ENgOeC9xaSSpT6QOXAJpeI1SFlKPSOMc6vojP7Vy/DaEGVUAbAbOBXdPrbYDu\nSi5S6cMJQGsYre98IxkuzrH6/Y/2uer1uV3Xb6UqTQDK6Qa6M3B/RPxN0muBlwILIuLxapdGcteM\nseIyq5XSfv8dHac23Nw+nq/fhlP1bqBkvX5WS9oF+BawHdl00GYtad25fbKEYGiQV9Fc12/VVM5I\n4MGIWC1pFvC1iPiapJtrHZiZrcujea3ayikBPCPp7cC7gP9J29wLyBpSX18f++3XyZZb7sJ++x0y\nroFYPT1z6eg4FVgALEizjM6teqzlcq7faqWcNoC9gPcBv4yIiyXtRLZGwFk1C8ptADYOfX19zJz5\nTp555otpy0fYcMPVXHrpf1dcf98oC7y4rt8qUfVF4Ye5wPbAsRHxhUqDq+AaTgCsYsPN2QPfpKvr\nhU0xb0+e5/Cx8ag0ASh3TeDnk60D8DbghcCi8YVnZmNxXb/Vy4htAJI2lXScpD7gV8DOwE4R8eKI\n6BnvBSXtLunm3OMJSR8a7/nMhvT0zGXDDU9hqO4+qwJaXmj9fSVc12/1NmIVkKRVZEs1fgK4JiJC\n0p0RsVPVLi5NAu4HDoyIe3PbXQVk49LX18dHP/p57r77PnbYYWs+//lP1LT+vlptBa7rt2qoWhuA\npJPIqnw2Ai4Gvg8srXIC0A18MiIOKdnuBMAaXjUGjLmu36qp6o3AaSTwsemxK3A6sCgifj+RQNO5\nLwB+HRH/VbLdCYA1vIkuFONcv1VbLRaEWRERn42IfYADgOcBV0wgRgAkbQi8AfjhRM9l1kxc12+N\noqxeQEMiYhmwDPhYFa79euDGiHh4uJ29vb1rnnd2dtLZ2VmFS5pVT0/PXAYG5rAqrZqdDRhbMOp7\n3MPHqqm/v5/+/v5xv7/icQDVIum/gSsiYr3/Ma4CsmZRbiOw6/qtHmo+EKwaJG0E3E3WrfSvw+x3\nAmAtw3X9Vi9VbwOQNDN116yaiHgqIqYNd/M3axWu67dGV86N/a3AHyV9QdIetQ7IrNb6+vro7p5N\nd/fscU0WV8557rjjDg477DAWLlzIwMAAPT09rvKxxlPOqjFkPX/eD1wH/BKYC2xSycozlTzwimBW\nI9VasnGk83iVLisS1V4RbIikacA7gZOA28nGBPxnRPxntRMltwFYrUy07/5o5zn44IuZMuXvruu3\nwtSiDeCNkhYB/WTrABwQEa8nWxryw+MN1GwiqlWNUx2DwJXccMNVruu35jJWEYFsVq3XjLDv8EqK\nG+U+cBWQjWIi1TjVrwL6QsAeMWnSlDjvvPMqPo9ZNVHNKiBJGwBXRkRnzVOida8bo8Vl7W2i1TjV\nmMBtcHCQE044gfPOO58dd9ydr3zlTI488siKz2NWTVVdDyCytYCflbRZRDw+8fDMijdjxowJzRCa\n79e/bNlvXN1jTaucbqBPAcskfVvS19Kj6g2/ZuUqas1e9+u3VlPObKDHDbM5YpgpHKrFVUA2lnqv\n2evRvNYMmmIqiLE4AbBGUckcPo2ykLy1r6qvCSxpN+BzwJ5AR9ocEfHi8YVo1hwqmbmzdHGYgYE5\nFS8OY1Zv5bQBXAh8E1gNdJJVvC6sYUxmZavFeIDx1PXPm3duuvnPAbKEYKg0YNaoylkPoCMirlRW\nL3M30CvpJrK1gs0KU4tct+frt3ZSTgngb5Imk00I90FJs8jWCTYrVDVz3RPt4VNUzySziSinBHAS\nMBX4EPAZYFPWjsAxa3rVyPXPmDGDRYsW5BqBXf9vjc+9gKxplVYBdXScWlEVkFfpslZTtW6gki7L\nvQxAuedExMwJBLkZcD6wVzrfeyLiutx+JwBWlvF2vaykX7+7d1qzqGYC0JmeHg1sDXyPLBF4G7Ay\nIk6aQJALgJ9HxAVpvqGNIuKJ3H4nAFYTleb6J1rKMKunqg8Ek3RjROw/1rYKAnwecPNo4wicAFgt\njGc0b7XWDzCrh6qvBwBMlbRz7gIvJmsUHq+dgIclXSjpJknnSZrI+cxG5Tl8zIZXTi+gk4GrJN2Z\nXu9ItiTkRK65H/DBiPiVpK8ApwGfnMA5zYY10R4+PT1zGRiYw6pV2euse2fNpsEyq6sxE4CIWJym\ng9iDrMF2eUT8fQLXvA+4LyJ+lV7/iCwBWEdvb++a552dnXR2dk7gktZuqtXDx907rZH19/fT398/\n7veX1Q1U0qvIqm42YG0voO+M+6LS1cB7I+L3knrJRhufmtvvNgAbN8/cae2qFmsCfw84GzgYeAVw\nQHpMxAnAQkm3kq0t/LkJns/Mdf1mFSqnDWB/YM9qZskj4lYmnoiYreE5fMwqV04voNuAbWodiNl4\nONdvNn7llACeD9wu6QZgqPE3JjIS2KwanOs3m5hyEoDe9O9QFZByz83qznP4mFVHOd1A+yVtTVZn\nH8ANEfFQzSMzG4Zz/WbVU04voLcA1wPHAG8BbpB0TK0DM8tzXb9Z9ZVTBfRx4IChXL+k5wM/A35Y\ny8DMhjjXb1Yb5fQCEvBw7vWjrJ0a2tpQLdbhHY5z/Wa1VU4JYDHQJ+kishv/W4ErahqVNaxarMM7\nHOf6zWpvzBJARJwCfItsxO4+wLci4t9rHZg1pmquwzsc5/rN6mfMEoCknYDLI+KS9LpD0o4RcVet\ng7P24ly/WX2V0wbwI+DZ3OvBtM3aUE/PXDo6TgUWAAvS9MgTmR28tXL99WofMauGclYEuyUiXlay\n7daI2LdmQXk20IZWzTVyW2nmTi8faUWrdDZQImLUB3Al8Mbc6zcCPxvrfRN5ZGFZ3uLFi6Ora1Z0\ndc2KxYsXFx3OhD377LNxzjnnxLRp0+Lss8+O1atXFx3ShHV1zQqYHxDpMT+6umYVHZa1kXTvLPte\nW04voPeTTd18Tnp9H/DO8tMkm6h69bypF9f1mzWIclMKYGNgk0pSl/E+cAlgHa2Ss2zFXH/e4sWL\no6Njq/S3mh8dHVu1RGnNmgfVLgGkeYA+C2wbEUdI2hOYHhHfrmG6ZC2mHXL9Xj7Smk05jcCLgQuB\n/4iIl0qaAtwcEXuP+6LSXcBfyHoX/SMiDizZH2PF1U6auXHRM3ea1U+ljcDlJAC/johXSLo5Il6e\ntq3XM6jCIO8E9o+IP4+w3wlAiWr2vKmXVurhY9YMKk0AymkEflLSlrkLvBJ4YjzBlfB8QhWYMWNG\nU9z0wbl+s2ZRTgLQA1wGvFjStWQrhL15gtcN4EpJz5JNLXHeBM9nDaId6vrNWkU5C8LcKOlQYOh/\n8u8i4h8TvO7BEfFgmlp6qaTlEXFN/oDe3t41zzs7O+ns7JzgJa2WnOs3q7/+/n76+/vH/f4R2wAk\nHQjcGxEPptdzgNnAXUDvSPX3FQcgnQ48GRHzctvcBtBE5s+fz4c/fAqDg4OcffZZvPe97y06JLO2\nVGkbwGhzAX2LtAi8pNcAZ5JNAPMXYNzTP0qaKmmT9HwjoBtYNt7zWXEGBwf5wAc+wLvf/R4ee+ww\nnnjibD70oY97DhyzJjFaFdCkXC7/rWR19ZcAl0i6dQLX3ApYJGno+gsjYskEzmcFGKrrv/nm3wCf\nA04DYNXMfzhrAAAORUlEQVSqScybd27TNFibtbPRSgCTU59/gMOBq3L7ymk8HlZE3BkRL0uPvSPi\n8+M9l9Vf6cydBxxwKLBN0WGZ2TiMdiO/GPi5pEeAp4FrACTtCjxeh9iswQzXw2fvvffmF7+Yw6pV\n2THZ9NALig3UzMoy6kAwSdOBrYElEfFU2rYbsHFE3FSzoNwI3FDG6uHTjIPUzFpR1UcCF8EJQOPw\naF6z5lHNXkDWxlpplS4zG964G3OtdXk0r1l7cAnA1nCu36y9uARggHP9Zu3IJYAG0NfXR3f3bLq7\nZ9d9FO1Yuf4iYzOz2nIvoIIVudjLWD18mnkhGrN25F5ATWbevHPTDXYOkN1sh/rU10q5df1FxGZm\n9eM2gDbjun4zG+ISQMF6eubS0XEq2USrC9JUCnOrfp3x9PCpV2xmVgy3ATSAWk+lMJHRvJ7mwax5\neCoIW8OrdJm1l1osCm9NqB3r+l1aMauM2wBaTLuO5h3qsrp06UyWLp3J0UfP8bgFszEUVgKQNBn4\nNXBfRLyhqDhaSTvm+oes22UVVq3CK5OZjaHIEsCJwO2AK/snqF1z/WY2MYWUACRtBxwJfBb4cBEx\ntIp2zvXn9fTMZWDAK5OZVaKoEsCXgVOAwYKu3/Sc61/XjBkzWLRoAV1dl9LVdamnrDArQ91LAJKO\nAh6KiJsldY50XG9v75rnnZ2ddHaOeGjbca5/eDNmzPBN39pKf38//f39435/3ccBSPoc8E5gNfBc\nYFPgkoh4V+4YjwMYhvv1m9lommogmKRDgY+U9gJyArA+r81rZmNpxtlAfacfhev6y+N1C8wq56kg\nGphz/eXxugVmmWYsAVgJ5/or43ULzMbHcwE1mKFc/8qVK9ltt5fT13cte++9t3OzZlZ1LgE0iHyu\nf9ddd+XOOx/l2mvf4XltyuB1C8zGx20ADaC0rv+EEz7G0qUzGZrXBrIBTkuWXFJkmA3NM4GaeTro\npuJ+/dXjQWBmlXMCUJDRRvN6Xhszqwe3AdRZOT18PK+NmdWD2wDqyP36zayWPA6gAblfv5k1IrcB\n1Jhn7jSzRuUSQI04129mjc4lgBpwrt/MmoFLAFXkXL+ZNROXAKqk3Fy/R6yaWaNwCWCCKsn1D01b\nvHTpTM/xY2aF8ziACai0X3939+w0x8/WwLnAA7z85ZO56aaBeoRrZi2u4ccBSHqupOsl3SLpNkm9\n9Y5hoiZW17+MbJK3mcD7ufXW210KMLNCFFICkDQ1Ip6WtAEwAJwYEdfn9jdsCWAio3n7+vo48sh3\nMDg4D8/0aWbV1vAlAICIeDo93RCYAgwWEUclqtHDZ8aMGey77941itDMrDKF9AKSNAm4CdgZOCci\nflVEHOWqZr/+z3/+o2n92uy1Z/o0s6IUkgBExCDwMknPAxZJ2isifps/pre3d83zzs5OOjs76xoj\n1Ga+/qGZPtd2BfVMn2Y2Pv39/fT394/7/YX3ApL0CeDpiJiX21Z4G4Bn7jSzZtPwbQCSpknaLD3v\nALqA/6t3HCPxaF4zaxdFVAFtAyyQNJksAfp+RFxeQBzr8Rw+ZtZO6l4CiIhlEbFfROwbEftExBn1\njqHURHP9fX19dHfPprt7tvv0m1nTaPu5gCaa6x+a3mHVqrMAGBiY4yUczawptO1cQNWq658379x0\n858DZAnBUA8fM7NG1pYlANf1m5m1WQmgFj18enrm0tFxKrAAWJAGds2tSrxmZrVU+DiA4dRiHEAt\n+/V7jn8zawSVjgNo+QSgFqN5zcwaUaUJQEu3Abiu38xsZC3ZBuDRvGZmY2u5EoBz/WZm5WmZEoBz\n/WZmlWmJEoBz/WZmlWvqEoBz/WZm49e0JQDn+s3MJqbpSgDO9ZuZVUdTlQCc6zczq54iVgR7kaSr\nJP1W0m2SPjTWe5zrNzOrviKqgP4BnBwRewGvBD4g6SUjHXzHHXdw2GGHsXDhQgYGBujp6SlkKoeJ\nLLxcK40YEzRmXI6pPI6pfI0aVyWKWBHsTxFxS3r+JNl6wC8sPa7Rcv2N+MduxJigMeNyTOVxTOVr\n1LgqUWgbgKQdgZcD15fuO+yww1zXb2ZWQ4X1ApK0MfAj4MRUElhHI+T6zcxaWSHTQUuaAvwPcEVE\nfGWY/Y03R7WZWRNo6PUAJIls+axHI+Lkul7czMzWKCIBOAS4GvgNMHTxj0bE4roGYmbW5hpyRTAz\nM6u9hpoKYjyDxOoQ03MlXS/plhRTb9ExDZE0WdLNki4rOhYASXdJ+k2K6Yai4wGQtJmkH0n6P0m3\nS3plA8S0e/qOhh5PNMhv/eT0G18m6SJJz2mAmE5M8dwm6cSCYrhA0kpJy3LbtpC0VNLvJS2RtFmD\nxHVMun8+K2m/sc7RUAkAFQ4Sq4eI+Bvw2oh4GfAy4AhJBxUZU86JwO2srUorWgCdEfHyiDiw6GCS\nrwKXR8RLgJeSjTspVET8Ln1HLwf2B54GFhUZk6RtgROA/SNiH2AycGzBMe0NvBc4ANgXOErSzgWE\nciFwRMm204ClEbEb8LP0ut6Gi2sZcDRZNfuYGioBKHeQWL1FxNPp6YbAFGCwwHAAkLQdcCRwPlB2\nq38dNEwskp4HvDoiLgCIiNUR8UTBYZU6HFgREfcWHQjZuKCpkjYApgL3FxzPHsD1EfG3iHgW+Dkw\nq95BRMQ1wGMlm2eSdWYh/fumugbF8HFFxPKI+H2552ioBCBvtEFi9SZpkqRbgJXAkoj4VdExAV8G\nTqEBEqOcAK6U9GtJ/1J0MMBOwMOSLpR0k6TzJE0tOqgSxwIXFR1ERNwPzAPuAR4AHo+IK4uNituA\nV6fqlqnAPwHbFRzTkK0iYmV6vhLYqshgxqshE4CxBonVW0QMpiqg7YCDJO1VZDySjgIeioibaaAc\nN3BwqtZ4PVn13asLjmcDYD/gvyJiP+ApiimqD0vShsAbgB82QCybk+VqdyQrdW8s6R1FxhQRy4Gz\ngCXAFcDNNFaGB4DIetI0SjVsRRouAUiDxC4BvhcRPyk6nrxUfXAV69e71durgJmS7gQuBl4n6TsF\nx0REPJj+fZisTrvodoD7gPtyJbYfkSUIjeL1wI3p+yra4cCdEfFoRKwGfkz2OytURFwQEa+IiEOB\nx4HfFR1TslLS1gCStgEeKjiecWmoBCANEvs2cPtwI4SLIGnaUAu/pA6gi4IbEiPiYxHxoojYiawK\n4X8j4l1FxiRpqqRN0vONgG6yBqnCRMSfgHsl7ZY2HQ78tsCQSr2NLAFvBHcDr5TUkf4fHk7WwaBQ\nkl6Q/t2erHGz8Oqy5FJgTno+B2iozGoyZu1Aoy0IczDwz8BvJN2cthU9SGwbYIGkyWQJ5vcj4vIC\n4xlOIxQ/twIWZfcONgAWRsSSYkMCsp4tC1N1ywrg3QXHA6xJJA8HGqGthIi4QdKPgJuA1enfc4uN\nCoAfSdqSrIfgv0XEX+odgKSLgUOBaZLuBT4JnAn8QNLxwF3AWxogrtOBPwNfA6YBP5V0c0S8fsRz\neCCYmVl7aqgqIDMzqx8nAGZmbcoJgJlZm3ICYGbWppwAmJm1KScAZmZtygmANZQ0je3NaQrgH6TB\nd+M913xJs9Pz80abWVbSoZKmj+Mad0naYpwxTZF0ZppS+EZJ10o6Iu3rl7Q8N2X089P250j6vqQ/\nSLpO0g6Vxmw2xAmANZqn01TJ+wDPAO/P70wzVZZrzRwtEfEvETHaCO7XMr6pDyodSJOfN+YzZAPo\n9oqI/clmlNwkd9zbh6aNzk0XcTzZcqq7kk0IeNY4YjYDnABYY7sG2CXlzq+R9P+A29LsrF+UdIOk\nWyXNhWwqEUnnpJzzUuAFQydKOer90/MjUo77lrSoxw7A+4CTU277YEnPV7aQzA3p8ar03i2VLQBy\nm6TzGGG4vaQnJX0pHXelpGkl+6eSzXV/QkT8AyAiHoqI/MRww507Pw3xJcBh6XzbSLo6V3o6pKJv\n2tqSEwBrSCmnfyTZ2tGQTQ3+oYjYg+zG+XhadOZA4F/S9OFHA7sBLwHexbo5+gAiVaWcC8xKM7we\nExF3A98EvpRy278gW0jmy+kabyZbdwGy4fZXR8TeZBPebT/CR5gK/Cod9/P0vjUfD9gFuGeM2W4X\npBv6x3PbtgXuhWx9A+CJNFXC24HFaTbWlwK3jHJeM6Dx5gIy68jNA3U1cAHZHFE3pBs1ZBPN7SPp\nzen1psCuwKuBi9L0vA9K+t+Sc4tspbmrh84VEY+X7B9yOPCSNLcRwCZp/p5XkyU0RMTlkkoXChky\nCHw/Pf8e2eyaeWNVHb0jIh5QNjX6JZLeGRHfHeHYAG4ALkiz6f4kIm4d4/xmTgCs4axKudg10k34\nqZLjPhgRS0uOO5KxZ0Ast85ewEER8cwwsVS6BoNKrhvAH4HtJW0SEX9dL8iIB9K/T0q6iKyk812y\nVbq2Bx5IpaTnRcSfgWuUrb9wFDBf0pdGSTDMAFcBWXPqA/5tqEFY0m6pTv1q4K2pjWAbsobdvACu\nA16TqozI9eD5K2sbYCFbhGTNQu2S9k1PryarbkHS64HNR4hxEnBMev52svaMtYFErCKb+vyrKddO\nand4s6TJQ20Gad8bWDu1dn4a4jeTrUc7NF3ywxFxPll11TqJqNlwnABYoxkuh1664tL5ZHPV3yRp\nGfANYHJELAL+kPYtAK5d70QRjwBzgR8rW+ZzaD7+y4CjhxqByW7+r0iNzL8layQG+BRZAnIbWVXQ\n3QzvKeDAFF8n8Olhjvk48DBwezruMuAJ4DnAYkm3kq2CdS9wXnrPt4EtJf0BOIm1K5x1ArdIuols\nauKvjhCX2RqeDtqsBiT9NSI2GftIs+K4BGBWG85ZWcNzCcDMrE25BGBm1qacAJiZtSknAGZmbcoJ\ngJlZm3ICYGbWppwAmJm1qf8PGt+Oop6ii3kAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f6cf6b3f110>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "task = \"pIC50\"\n",
    "dnn_predicted_test = np.array(dnn_test_df[task + \"_pred\"])\n",
    "dnn_true_test = np.array(dnn_test_df[task])\n",
    "plt.scatter(dnn_predicted_test, dnn_true_test)\n",
    "plt.xlabel('Predicted pIC50s')\n",
    "plt.ylabel('Secondary Assay')\n",
    "plt.title(r'DNN predicted IC50 vs. Secondary Assay')\n",
    "plt.xlim([2, 11])\n",
    "plt.ylim([2, 11])\n",
    "plt.plot([2, 11], [2, 11], color='k')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we construct classification models for the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Make directories to store the raw and featurized datasets.\n",
    "c_feature_dir = tempfile.mkdtemp()\n",
    "c_samples_dir = tempfile.mkdtemp()\n",
    "c_crystal_feature_dir = tempfile.mkdtemp()\n",
    "c_crystal_samples_dir = tempfile.mkdtemp()\n",
    "c_train_dir = tempfile.mkdtemp()\n",
    "c_valid_dir = tempfile.mkdtemp()\n",
    "c_test_dir = tempfile.mkdtemp()\n",
    "c_crystal_dir = tempfile.mkdtemp()\n",
    "\n",
    "c_featurizer = DataFeaturizer(tasks=[\"Class\"],\n",
    "                            smiles_field=\"mol\",\n",
    "                            id_field=\"CID\",\n",
    "                            user_specified_features=user_specified_features,\n",
    "                            split_field=\"Model\")\n",
    "c_featurized_samples = c_featurizer.featurize(\n",
    "    dataset_file, c_feature_dir, c_samples_dir, shard_size=2000)\n",
    "c_crystal_featurized_samples = c_featurizer.featurize(\n",
    "    crystal_dataset_file, c_crystal_feature_dir, c_crystal_samples_dir, shard_size=2000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "c_train_samples, c_valid_samples, c_test_samples = c_featurized_samples.train_valid_test_split(\n",
    "    splittype, c_train_dir, c_valid_dir, c_test_dir)\n",
    "#NOTE THE RENAMING:\n",
    "c_valid_samples, c_test_samples = c_test_samples, c_valid_samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of compounds in c_train set\n",
      "204\n",
      "Number of compounds in c_validation set\n",
      "1273\n",
      "Number of compounds in c_test set\n",
      "45\n",
      "Number of compounds in c_crystal set\n",
      "25\n"
     ]
    }
   ],
   "source": [
    "from deepchem.utils.dataset import Dataset\n",
    "c_train_dataset = Dataset(data_dir=c_train_dir, samples=c_train_samples, \n",
    "                          featurizers=[], tasks=[\"Class\"],\n",
    "                          use_user_specified_features=True)\n",
    "c_valid_dataset = Dataset(data_dir=c_valid_dir, samples=c_valid_samples, \n",
    "                          featurizers=[], tasks=[\"Class\"],\n",
    "                          use_user_specified_features=True)\n",
    "c_test_dataset = Dataset(data_dir=c_test_dir, samples=c_test_samples, \n",
    "                         featurizers=[], tasks=[\"Class\"],\n",
    "                         use_user_specified_features=True)\n",
    "c_crystal_dataset = Dataset(data_dir=c_crystal_dir, samples=c_crystal_featurized_samples, \n",
    "                            featurizers=[], tasks=[\"Class\"],\n",
    "                            use_user_specified_features=True)\n",
    "print(\"Number of compounds in c_train set\")\n",
    "print(len(c_train_dataset))\n",
    "print(\"Number of compounds in c_validation set\")\n",
    "print(len(c_valid_dataset))\n",
    "print(\"Number of compounds in c_test set\")\n",
    "print(len(c_test_dataset))\n",
    "print(\"Number of compounds in c_crystal set\")\n",
    "print(len(c_crystal_dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from deepchem.transformers import NormalizationTransformer\n",
    "from deepchem.transformers import ClippingTransformer\n",
    "\n",
    "\n",
    "c_input_transformers = [NormalizationTransformer(transform_X=True, dataset=train_dataset),\n",
    "                      ClippingTransformer(transform_X=True, dataset=train_dataset)]\n",
    "c_output_transformers = []\n",
    "c_transformers = c_input_transformers + c_output_transformers\n",
    "for transformer in c_transformers:\n",
    "    transformer.transform(c_train_dataset)\n",
    "for transformer in c_transformers:\n",
    "    transformer.transform(c_valid_dataset)\n",
    "for transformer in c_transformers:\n",
    "    transformer.transform(c_test_dataset)\n",
    "for transformer in c_transformers:\n",
    "    transformer.transform(c_crystal_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model 0/8, Metric roc_auc_score, Validation set 0: 0.718956\n",
      "\tbest_validation_score so  far: -inf\n",
      "Model 1/8, Metric roc_auc_score, Validation set 1: 0.681130\n",
      "\tbest_validation_score so  far: 0.718956\n",
      "Model 2/8, Metric roc_auc_score, Validation set 2: 0.673560\n",
      "\tbest_validation_score so  far: 0.718956\n",
      "Model 3/8, Metric roc_auc_score, Validation set 3: 0.687243\n",
      "\tbest_validation_score so  far: 0.718956\n",
      "Model 4/8, Metric roc_auc_score, Validation set 4: 0.699159\n",
      "\tbest_validation_score so  far: 0.718956\n",
      "Model 5/8, Metric roc_auc_score, Validation set 5: 0.710670\n",
      "\tbest_validation_score so  far: 0.718956\n",
      "Model 6/8, Metric roc_auc_score, Validation set 6: 0.696389\n",
      "\tbest_validation_score so  far: 0.718956\n",
      "Model 7/8, Metric roc_auc_score, Validation set 7: 0.702598\n",
      "\tbest_validation_score so  far: 0.718956\n",
      "Best hyperparameters: [('n_estimators', 10), ('data_shape', (589,)), ('max_features', 'auto')]\n",
      "train_score: 0.970588\n",
      "validation_score: 0.718956\n"
     ]
    }
   ],
   "source": [
    "from deepchem.hyperparameters import HyperparamOpt\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from deepchem.models.standard import SklearnModel\n",
    "\n",
    "def model_builder(task_types, params_dict, verbosity):\n",
    "    n_estimators = params_dict[\"n_estimators\"]\n",
    "    max_features = params_dict[\"max_features\"]\n",
    "    return SklearnModel(\n",
    "        task_types, params_dict,\n",
    "        model_instance=RandomForestClassifier(n_estimators=n_estimators,\n",
    "                                              max_features=max_features))\n",
    "params_dict = {\n",
    "    \"n_estimators\": [10, 100],\n",
    "    \"data_shape\": [c_train_dataset.get_data_shape()],\n",
    "    \"max_features\": [\"auto\", \"sqrt\", \"log2\", None],\n",
    "    }\n",
    "c_optimizer = HyperparamOpt(model_builder, {\"Class\": \"classification\"})\n",
    "c_best_rf, c_best_rf_hyperparams, c_all_rf_results = c_optimizer.hyperparam_search(\n",
    "    params_dict, c_train_dataset, c_valid_dataset, c_output_transformers, metric=\"roc_auc_score\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model 0/25, Metric roc_auc_score, Validation set 0: 0.714419\n",
      "\tbest_validation_score so  far: -inf\n",
      "Model 1/25, Metric roc_auc_score, Validation set 1: 0.675208\n",
      "\tbest_validation_score so  far: 0.714419\n",
      "Model 2/25, Metric roc_auc_score, Validation set 2: 0.500000\n",
      "\tbest_validation_score so  far: 0.714419\n",
      "Model 3/25, Metric roc_auc_score, Validation set 3: 0.595066\n",
      "\tbest_validation_score so  far: 0.714419\n",
      "Model 4/25, Metric roc_auc_score, Validation set 4: 0.619281\n",
      "\tbest_validation_score so  far: 0.714419\n",
      "Model 5/25, Metric roc_auc_score, Validation set 5: 0.710980\n",
      "\tbest_validation_score so  far: 0.714419\n",
      "Model 6/25, Metric roc_auc_score, Validation set 6: 0.684616\n",
      "\tbest_validation_score so  far: 0.714419\n",
      "Model 7/25, Metric roc_auc_score, Validation set 7: 0.500000\n",
      "\tbest_validation_score so  far: 0.714419\n",
      "Model 8/25, Metric roc_auc_score, Validation set 8: 0.609538\n",
      "\tbest_validation_score so  far: 0.714419\n",
      "Model 9/25, Metric roc_auc_score, Validation set 9: 0.651710\n",
      "\tbest_validation_score so  far: 0.714419\n",
      "Model 10/25, Metric roc_auc_score, Validation set 10: 0.716974\n",
      "\tbest_validation_score so  far: 0.714419\n",
      "Model 11/25, Metric roc_auc_score, Validation set 11: 0.692974\n",
      "\tbest_validation_score so  far: 0.716974\n",
      "Model 12/25, Metric roc_auc_score, Validation set 12: 0.500000\n",
      "\tbest_validation_score so  far: 0.716974\n",
      "Model 13/25, Metric roc_auc_score, Validation set 13: 0.623293\n",
      "\tbest_validation_score so  far: 0.716974\n",
      "Model 14/25, Metric roc_auc_score, Validation set 14: 0.643376\n",
      "\tbest_validation_score so  far: 0.716974\n",
      "Model 15/25, Metric roc_auc_score, Validation set 15: 0.709810\n",
      "\tbest_validation_score so  far: 0.716974\n",
      "Model 16/25, Metric roc_auc_score, Validation set 16: 0.675518\n",
      "\tbest_validation_score so  far: 0.716974\n",
      "Model 17/25, Metric roc_auc_score, Validation set 17: 0.500000\n",
      "\tbest_validation_score so  far: 0.716974\n",
      "Model 18/25, Metric roc_auc_score, Validation set 18: 0.660880\n",
      "\tbest_validation_score so  far: 0.716974\n",
      "Model 19/25, Metric roc_auc_score, Validation set 19: 0.640247\n",
      "\tbest_validation_score so  far: 0.716974\n",
      "Model 20/25, Metric roc_auc_score, Validation set 20: 0.703291\n",
      "\tbest_validation_score so  far: 0.716974\n",
      "Model 21/25, Metric roc_auc_score, Validation set 21: 0.668712\n",
      "\tbest_validation_score so  far: 0.716974\n",
      "Model 22/25, Metric roc_auc_score, Validation set 22: 0.500000\n",
      "\tbest_validation_score so  far: 0.716974\n",
      "Model 23/25, Metric roc_auc_score, Validation set 23: 0.660784\n",
      "\tbest_validation_score so  far: 0.716974\n",
      "Model 24/25, Metric roc_auc_score, Validation set 24: 0.647196\n",
      "\tbest_validation_score so  far: 0.716974\n",
      "Best hyperparameters: [('data_shape', (589,)), ('activation', 'relu'), ('batch_size', 50), ('nb_layers', 1), ('decay', 1.90851111643045e-06), ('nesterov', False), ('nb_epoch', 40), ('learning_rate', 0.00062478423019296655), ('init', 'glorot_uniform'), ('nb_hidden', 1000), ('batchnorm', False), ('dropout', 0.5), ('momentum', 0.9)]\n",
      "train_score: 0.882353\n",
      "validation_score: 0.716974\n"
     ]
    }
   ],
   "source": [
    "from deepchem.models.deep import SingleTaskDNN\n",
    "import numpy.random\n",
    "\n",
    "params_dict = {\"activation\": [\"relu\"],\n",
    "                \"momentum\": [.9],\n",
    "                \"batch_size\": [50],\n",
    "                \"init\": [\"glorot_uniform\"],\n",
    "                \"data_shape\": [train_dataset.get_data_shape()],\n",
    "                \"learning_rate\": np.power(10., np.random.uniform(-5, -2, size=5)),\n",
    "                \"decay\": np.power(10, np.random.uniform(-6, -4, size=5)),\n",
    "                \"nb_hidden\": [1000],\n",
    "                \"nb_epoch\": [40],\n",
    "                \"nesterov\": [False],\n",
    "                \"dropout\": [.5],\n",
    "                \"nb_layers\": [1],\n",
    "                \"batchnorm\": [False],\n",
    "              }\n",
    "\n",
    "c_optimizer = HyperparamOpt(SingleTaskDNN, {\"Class\": \"classification\"})\n",
    "c_best_dnn, c_best_hyperparams, c_all_results = c_optimizer.hyperparam_search(\n",
    "    params_dict, c_train_dataset, c_valid_dataset, c_output_transformers, metric=\"roc_auc_score\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RF Train set ROC-AUC 0.970588\n",
      "RF Valid set ROC-AUC 0.718956\n",
      "RF Test set ROC-AUC 0.215909\n",
      "RF Crystal set ROC-AUC 0.500000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rbharath/deepchem/deepchem/utils/evaluate.py:42: UserWarning: ROC AUC score calculation failed.\n",
      "  warnings.warn(\"ROC AUC score calculation failed.\")\n"
     ]
    }
   ],
   "source": [
    "c_rf_train_csv_out = \"rf_train_classifier.csv\"\n",
    "c_rf_train_stats_out = \"rf_train_stats_classifier.txt\"\n",
    "c_rf_train_evaluator = Evaluator(c_best_rf, c_train_dataset, c_output_transformers)\n",
    "c_rf_train_df, c_rf_train_roc_auc_score = c_rf_train_evaluator.compute_model_performance(\n",
    "    c_rf_train_csv_out, c_rf_train_stats_out)\n",
    "c_rf_train_roc_auc_score = c_rf_train_roc_auc_score.iloc[0][\"roc_auc_score\"]\n",
    "print(\"RF Train set ROC-AUC %f\" % (c_rf_train_roc_auc_score))\n",
    "\n",
    "c_rf_valid_csv_out = \"rf_valid_classifier.csv\"\n",
    "c_rf_valid_stats_out = \"rf_valid_stats_classifier.txt\"\n",
    "c_rf_valid_evaluator = Evaluator(c_best_rf, c_valid_dataset, c_output_transformers)\n",
    "c_rf_valid_df, c_rf_valid_roc_auc_score = c_rf_valid_evaluator.compute_model_performance(\n",
    "    c_rf_valid_csv_out, c_rf_valid_stats_out)\n",
    "c_rf_valid_roc_auc_score = c_rf_valid_roc_auc_score.iloc[0][\"roc_auc_score\"]\n",
    "print(\"RF Valid set ROC-AUC %f\" % (c_rf_valid_roc_auc_score))\n",
    "\n",
    "c_rf_test_csv_out = \"rf_test_classifier.csv\"\n",
    "c_rf_test_stats_out = \"rf_test_stats_classifier.txt\"\n",
    "c_rf_test_evaluator = Evaluator(c_best_rf, c_test_dataset, c_output_transformers)\n",
    "c_rf_test_df, c_rf_test_roc_auc_score = c_rf_test_evaluator.compute_model_performance(\n",
    "    c_rf_test_csv_out, c_rf_test_stats_out)\n",
    "c_rf_test_roc_auc_score = c_rf_test_roc_auc_score.iloc[0][\"roc_auc_score\"]\n",
    "print(\"RF Test set ROC-AUC %f\" % (c_rf_test_roc_auc_score))\n",
    "\n",
    "c_rf_crystal_csv_out = \"rf_crystal_classifier.csv\"\n",
    "c_rf_crystal_stats_out = \"rf_crystal_stats_classifier.txt\"\n",
    "c_rf_crystal_evaluator = Evaluator(c_best_rf, c_crystal_dataset, c_output_transformers)\n",
    "c_rf_crystal_df, c_rf_crystal_roc_auc_score = c_rf_crystal_evaluator.compute_model_performance(\n",
    "    c_rf_crystal_csv_out, c_rf_crystal_stats_out)\n",
    "c_rf_crystal_roc_auc_score = c_rf_crystal_roc_auc_score.iloc[0][\"roc_auc_score\"]\n",
    "print(\"RF Crystal set ROC-AUC %f\" % (c_rf_crystal_roc_auc_score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DNN Train set ROC-AUC 0.882353\n",
      "DNN Valid set ROC-AUC 0.716974\n",
      "DNN Test set ROC-AUC 0.215909\n",
      "DNN Crystal set ROC-AUC 0.500000\n"
     ]
    }
   ],
   "source": [
    "c_dnn_train_csv_out = \"c_dnn_train_classifier.csv\"\n",
    "c_dnn_train_stats_out = \"c_dnn_train_stats_classifier.txt\"                                   \n",
    "c_dnn_train_evaluator = Evaluator(c_best_dnn, c_train_dataset, c_output_transformers)            \n",
    "c_dnn_train_df, c_dnn_train_roc_auc_score = c_dnn_train_evaluator.compute_model_performance(\n",
    "    c_dnn_train_csv_out, c_dnn_train_stats_out)\n",
    "c_dnn_train_roc_auc_score = c_dnn_train_roc_auc_score.iloc[0][\"roc_auc_score\"]               \n",
    "print(\"DNN Train set ROC-AUC %f\" % (c_dnn_train_roc_auc_score))\n",
    "\n",
    "c_dnn_valid_csv_out = \"c_dnn_valid_classifier.csv\"                                           \n",
    "c_dnn_valid_stats_out = \"c_dnn_valid_stats_classifier.txt\"                                   \n",
    "c_dnn_valid_evaluator = Evaluator(c_best_dnn, c_valid_dataset, c_output_transformers)            \n",
    "c_dnn_valid_df, c_dnn_valid_roc_auc_score = c_dnn_valid_evaluator.compute_model_performance(\n",
    "    c_dnn_valid_csv_out, c_dnn_valid_stats_out)\n",
    "c_dnn_valid_roc_auc_score = c_dnn_valid_roc_auc_score.iloc[0][\"roc_auc_score\"]               \n",
    "print(\"DNN Valid set ROC-AUC %f\" % (c_dnn_valid_roc_auc_score))\n",
    "                                                                                             \n",
    "c_dnn_test_csv_out = \"c_dnn_test_classifier.csv\"\n",
    "c_dnn_test_stats_out = \"c_dnn_test_stats_classifier.txt\"                                                                                                                                \n",
    "c_dnn_test_evaluator = Evaluator(c_best_dnn, c_test_dataset, c_output_transformers)                                                                                                         \n",
    "c_dnn_test_df, c_dnn_test_roc_auc_score = c_dnn_test_evaluator.compute_model_performance(                                                                                          \n",
    "    c_dnn_test_csv_out, c_dnn_test_stats_out)\n",
    "c_dnn_test_roc_auc_score = c_dnn_test_roc_auc_score.iloc[0][\"roc_auc_score\"]                                                                                                            \n",
    "print(\"DNN Test set ROC-AUC %f\" % (c_dnn_test_roc_auc_score))\n",
    "\n",
    "c_dnn_crystal_csv_out = \"c_dnn_crystal_classifier.csv\"\n",
    "c_dnn_crystal_stats_out = \"c_dnn_crystal_stats_classifier.txt\"                                                                                                                          \n",
    "c_dnn_crystal_evaluator = Evaluator(c_best_dnn, c_crystal_dataset, c_output_transformers)                                                                                                   \n",
    "c_dnn_crystal_df, c_dnn_crystal_roc_auc_score = c_dnn_crystal_evaluator.compute_model_performance(\n",
    "    c_dnn_crystal_csv_out, c_dnn_crystal_stats_out)\n",
    "c_dnn_crystal_roc_auc_score = c_dnn_crystal_roc_auc_score.iloc[0][\"roc_auc_score\"]                                                                                                      \n",
    "print(\"DNN Crystal set ROC-AUC %f\" % (c_dnn_crystal_roc_auc_score))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We notice that the trained DNN classifier doesn't generalize well. The DNN requires the regression values to learn meaningful models. Thus, we use the trained DNN regressor and threshold to obtain classification results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DNN Train set ROC-AUC 0.632353\n",
      "DNN Valid set ROC-AUC 0.634492\n",
      "DNN Test set ROC-AUC 0.545455\n",
      "DNN Crystal set ROC-AUC 0.500000\n"
     ]
    }
   ],
   "source": [
    "threshold = 7.\n",
    "\n",
    "dnn_train_csv_out = \"dnn_train_classifier.csv\"    \n",
    "dnn_train_stats_out = \"dnn_train_stats_classifier.txt\"                                       \n",
    "dnn_train_evaluator = Evaluator(best_dnn, c_train_dataset, output_transformers)\n",
    "dnn_train_df, dnn_train_roc_auc_score = dnn_train_evaluator.compute_model_performance(      \n",
    "    dnn_train_csv_out, dnn_train_stats_out, threshold=threshold)\n",
    "dnn_train_roc_auc_score = dnn_train_roc_auc_score.iloc[0][\"roc_auc_score\"]                   \n",
    "print(\"DNN Train set ROC-AUC %f\" % (dnn_train_roc_auc_score))\n",
    "\n",
    "dnn_valid_csv_out = \"dnn_valid_classifier.csv\"                                               \n",
    "dnn_valid_stats_out = \"dnn_valid_stats_classifier.txt\"                                       \n",
    "dnn_valid_evaluator = Evaluator(best_dnn, c_valid_dataset, output_transformers)                \n",
    "dnn_valid_df, dnn_valid_roc_auc_score = dnn_valid_evaluator.compute_model_performance(      \n",
    "    dnn_valid_csv_out, dnn_valid_stats_out, threshold=threshold)\n",
    "dnn_valid_roc_auc_score = dnn_valid_roc_auc_score.iloc[0][\"roc_auc_score\"]                   \n",
    "print(\"DNN Valid set ROC-AUC %f\" % (dnn_valid_roc_auc_score))\n",
    "                                                                                             \n",
    "dnn_test_csv_out = \"dnn_test_classifier.csv\"\n",
    "dnn_test_stats_out = \"dnn_test_stats_classifier.txt\"                                                                                                                                    \n",
    "dnn_test_evaluator = Evaluator(best_dnn, c_test_dataset, output_transformers)                                                                                                             \n",
    "dnn_test_df, dnn_test_roc_auc_score = dnn_test_evaluator.compute_model_performance(                                                                                                \n",
    "    dnn_test_csv_out, dnn_test_stats_out, threshold=threshold)\n",
    "dnn_test_roc_auc_score = dnn_test_roc_auc_score.iloc[0][\"roc_auc_score\"]                                                                                                                \n",
    "print(\"DNN Test set ROC-AUC %f\" % (dnn_test_roc_auc_score))\n",
    "\n",
    "dnn_crystal_csv_out = \"dnn_crystal_classifier.csv\"\n",
    "dnn_crystal_stats_out = \"dnn_crystal_stats_classifier.txt\"                                                                                                                              \n",
    "dnn_crystal_evaluator = Evaluator(best_dnn, c_crystal_dataset, output_transformers)                                                                                                       \n",
    "dnn_crystal_df, dnn_crystal_roc_auc_score = dnn_crystal_evaluator.compute_model_performance(\n",
    "    dnn_crystal_csv_out, dnn_crystal_stats_out, threshold=threshold)\n",
    "dnn_crystal_roc_auc_score = dnn_crystal_roc_auc_score.iloc[0][\"roc_auc_score\"]                                                                                                          \n",
    "print(\"DNN Crystal set ROC-AUC %f\" % (dnn_crystal_roc_auc_score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}