Solution.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "from nltk.tokenize import word_tokenize,sent_tokenize\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data = pd.read_table('Data/training_set_rel3.tsv',encoding = \"ISO-8859-1\")\n",
    "train_data_except_set2 = train_data.loc[train_data.essay_set != 2,['essay_set','essay','domain1_score']]\n",
    "train_data_except_set2.columns = ['essay_set','essay','score']\n",
    "train_data_set2_1 = train_data.loc[train_data['essay_set'] == 2 ,['essay_set','essay','domain1_score']]\n",
    "train_data_set2_1['essay_set'] = \"2_1\"\n",
    "train_data_set2_1.columns = ['essay_set','essay','score']\n",
    "train_data_set2_2 = train_data.loc[train_data['essay_set'] == 2 ,['essay_set','essay','domain2_score']]\n",
    "train_data_set2_2['essay_set'] = \"2_2\"\n",
    "train_data_set2_2.columns = ['essay_set','essay','score']\n",
    "train_data_format = pd.concat([train_data_except_set2,train_data_set2_2,train_data_set2_2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_distinct_word_count(text):\n",
    "    word_list = word_tokenize(text)\n",
    "    return len(set(word_list))\n",
    "\n",
    "def get_word_count_token(text):\n",
    "    word_list = word_tokenize(text)\n",
    "    return len(word_list)\n",
    "\n",
    "def get_sentence_count(text):\n",
    "    sent_list = sent_tokenize(text)\n",
    "    return len(sent_list)\n",
    "\n",
    "def get_character_count(text):\n",
    "    return len(text)\n",
    "\n",
    "def get_word_count(text):\n",
    "    return len(re.findall(r\"\\s\", text))+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_data_format['length'] = train_data_format['essay'].apply(get_character_count)\n",
    "train_data_format['word_count'] = train_data_format['essay'].apply(get_word_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data_format['distinct_word'] = train_data_format['essay'].apply(get_distinct_word_count)\n",
    "train_data_format['sentence_count'] = train_data_format['essay'].apply(get_sentence_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>essay_set</th>\n",
       "      <th>essay</th>\n",
       "      <th>score</th>\n",
       "      <th>length</th>\n",
       "      <th>word_count</th>\n",
       "      <th>distinct_word</th>\n",
       "      <th>sentence_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Dear local newspaper, I think effects computer...</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1875</td>\n",
       "      <td>338</td>\n",
       "      <td>181</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  essay_set                                              essay  score  length  \\\n",
       "0         1  Dear local newspaper, I think effects computer...    8.0    1875   \n",
       "\n",
       "   word_count  distinct_word  sentence_count  \n",
       "0         338            181              16  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data_format.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_data = pd.read_table('Data/valid_set.tsv',encoding = \"ISO-8859-1\")\n",
    "test_data_except_set2 = test_data.loc[test_data.essay_set != 2,['essay_set','essay','domain1_predictionid']]\n",
    "test_data_except_set2.columns = ['essay_set','essay','predictionid']\n",
    "test_data_set2_1 = test_data.loc[test_data['essay_set'] == 2 ,['essay_set','essay','domain1_predictionid']]\n",
    "test_data_set2_1['essay_set'] = \"2_1\"\n",
    "test_data_set2_1.columns = ['essay_set','essay','predictionid']\n",
    "test_data_set2_2 = test_data.loc[test_data['essay_set'] == 2 ,['essay_set','essay','domain2_predictionid']]\n",
    "test_data_set2_2['essay_set'] = \"2_2\"\n",
    "test_data_set2_2.columns = ['essay_set','essay','predictionid']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_data_format = pd.concat([test_data_except_set2,test_data_set2_1,test_data_set2_2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_data_format['word_count'] = test_data_format['essay'].apply(get_word_count)\n",
    "test_data_format['length'] = test_data_format['essay'].apply(get_character_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_data_format['distinct_word'] = test_data_format['essay'].apply(get_distinct_word_count)\n",
    "test_data_format['sentence_count'] = test_data_format['essay'].apply(get_sentence_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>essay_set</th>\n",
       "      <th>essay</th>\n",
       "      <th>score</th>\n",
       "      <th>length</th>\n",
       "      <th>word_count</th>\n",
       "      <th>distinct_word</th>\n",
       "      <th>sentence_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Dear local newspaper, I think effects computer...</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1875</td>\n",
       "      <td>338</td>\n",
       "      <td>181</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  essay_set                                              essay  score  length  \\\n",
       "0         1  Dear local newspaper, I think effects computer...    8.0    1875   \n",
       "\n",
       "   word_count  distinct_word  sentence_count  \n",
       "0         338            181              16  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data_format.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>essay_set</th>\n",
       "      <th>essay</th>\n",
       "      <th>predictionid</th>\n",
       "      <th>word_count</th>\n",
       "      <th>length</th>\n",
       "      <th>distinct_word</th>\n",
       "      <th>sentence_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Dear @ORGANIZATION1, @CAPS1 more and more peop...</td>\n",
       "      <td>1788.0</td>\n",
       "      <td>230</td>\n",
       "      <td>1365</td>\n",
       "      <td>135</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  essay_set                                              essay  predictionid  \\\n",
       "0         1  Dear @ORGANIZATION1, @CAPS1 more and more peop...        1788.0   \n",
       "\n",
       "   word_count  length  distinct_word  sentence_count  \n",
       "0         230    1365            135              13  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data_format.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sort_by_id = test_data_format.sort_values(by=['predictionid'], axis=0,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X = train_data_format[['length','word_count']]\n",
    "y = train_data_format[['score']]\n",
    "X_test = test_data_format[['length','word_count']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(14776, 2)\n",
      "(14776, 1)\n",
      "(4818, 2)\n"
     ]
    }
   ],
   "source": [
    "print X.shape\n",
    "print y.shape\n",
    "print X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>length</th>\n",
       "      <th>word_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1875</td>\n",
       "      <td>338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2288</td>\n",
       "      <td>419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1541</td>\n",
       "      <td>279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3165</td>\n",
       "      <td>524</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2569</td>\n",
       "      <td>465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1276</td>\n",
       "      <td>246</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2808</td>\n",
       "      <td>499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2724</td>\n",
       "      <td>482</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2402</td>\n",
       "      <td>443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2632</td>\n",
       "      <td>502</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   length  word_count\n",
       "0    1875         338\n",
       "1    2288         419\n",
       "2    1541         279\n",
       "3    3165         524\n",
       "4    2569         465\n",
       "5    1276         246\n",
       "6    2808         499\n",
       "7    2724         482\n",
       "8    2402         443\n",
       "9    2632         502"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   score\n",
       "0    8.0\n",
       "1    9.0\n",
       "2    7.0\n",
       "3   10.0\n",
       "4    8.0\n",
       "5    8.0\n",
       "6   10.0\n",
       "7   10.0\n",
       "8    9.0\n",
       "9    9.0"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>length</th>\n",
       "      <th>word_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1365</td>\n",
       "      <td>230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1605</td>\n",
       "      <td>310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2343</td>\n",
       "      <td>441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1981</td>\n",
       "      <td>361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2517</td>\n",
       "      <td>453</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2471</td>\n",
       "      <td>460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2435</td>\n",
       "      <td>449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>3064</td>\n",
       "      <td>555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1549</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>3036</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   length  word_count\n",
       "0    1365         230\n",
       "1    1605         310\n",
       "2    2343         441\n",
       "3    1981         361\n",
       "4    2517         453\n",
       "5    2471         460\n",
       "6    2435         449\n",
       "7    3064         555\n",
       "8    1549         295\n",
       "9    3036         554"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py:2: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    }
   ],
   "source": [
    "randomForest = RandomForestRegressor(n_estimators=100)\n",
    "randomForest.fit(X,y)\n",
    "scores = randomForest.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "score_round = [round(i) for i in scores]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[3.0, 7.0, 11.0, 13.0, 8.0, 9.0, 8.0, 5.0, 6.0, 7.0]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score_round[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "predictions= zip(test_data_format['predictionid'],score_round)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}