data_prep.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from nltk.corpus import stopwords\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.tokenize import word_tokenize\n",
    "from tqdm.notebook import tqdm\n",
    "import copy\n",
    "import arff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /home/shubham/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to /home/shubham/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('stopwords')\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv('data_prep.csv',encoding = \"ISO-8859-1\") #training_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>#AUTHID</th>\n",
       "      <th>TEXT</th>\n",
       "      <th>cEXT</th>\n",
       "      <th>cNEU</th>\n",
       "      <th>cAGR</th>\n",
       "      <th>cCON</th>\n",
       "      <th>cOPN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1997_504851.txt</td>\n",
       "      <td>well right i woke midday nap. it sort weird ev...</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "      <td>y</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1997_605191.txt</td>\n",
       "      <td>well go stream conscious essay. i use thing li...</td>\n",
       "      <td>n</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "      <td>n</td>\n",
       "      <td>n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1997_687252.txt</td>\n",
       "      <td>an open keyboard button push. the thing final ...</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "      <td>y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1997_568848.txt</td>\n",
       "      <td>i cant believ it realli happen my puls race li...</td>\n",
       "      <td>y</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "      <td>y</td>\n",
       "      <td>n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1997_688160.txt</td>\n",
       "      <td>well i go good old stream conscious assign aga...</td>\n",
       "      <td>y</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2462</th>\n",
       "      <td>2462</td>\n",
       "      <td>2004_493.txt</td>\n",
       "      <td>im home. want go bed rememb i psycholog homewo...</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "      <td>n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2463</th>\n",
       "      <td>2463</td>\n",
       "      <td>2004_494.txt</td>\n",
       "      <td>stream consiousnesssskdj. how spell fuck i kno...</td>\n",
       "      <td>y</td>\n",
       "      <td>y</td>\n",
       "      <td>n</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2464</th>\n",
       "      <td>2464</td>\n",
       "      <td>2004_497.txt</td>\n",
       "      <td>it wednesday decemb th lot go semester. i tri ...</td>\n",
       "      <td>n</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "      <td>n</td>\n",
       "      <td>n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2465</th>\n",
       "      <td>2465</td>\n",
       "      <td>2004_498.txt</td>\n",
       "      <td>man week hellish. anyway time minut write assi...</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "      <td>n</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2466</th>\n",
       "      <td>2466</td>\n",
       "      <td>2004_499.txt</td>\n",
       "      <td>i gotten phone brady. im tri decid exacli week...</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "      <td>y</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2467 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0          #AUTHID  \\\n",
       "0              0  1997_504851.txt   \n",
       "1              1  1997_605191.txt   \n",
       "2              2  1997_687252.txt   \n",
       "3              3  1997_568848.txt   \n",
       "4              4  1997_688160.txt   \n",
       "...          ...              ...   \n",
       "2462        2462     2004_493.txt   \n",
       "2463        2463     2004_494.txt   \n",
       "2464        2464     2004_497.txt   \n",
       "2465        2465     2004_498.txt   \n",
       "2466        2466     2004_499.txt   \n",
       "\n",
       "                                                   TEXT cEXT cNEU cAGR cCON  \\\n",
       "0     well right i woke midday nap. it sort weird ev...    n    y    y    n   \n",
       "1     well go stream conscious essay. i use thing li...    n    n    y    n   \n",
       "2     an open keyboard button push. the thing final ...    n    y    n    y   \n",
       "3     i cant believ it realli happen my puls race li...    y    n    y    y   \n",
       "4     well i go good old stream conscious assign aga...    y    n    y    n   \n",
       "...                                                 ...  ...  ...  ...  ...   \n",
       "2462  im home. want go bed rememb i psycholog homewo...    n    y    n    y   \n",
       "2463  stream consiousnesssskdj. how spell fuck i kno...    y    y    n    n   \n",
       "2464  it wednesday decemb th lot go semester. i tri ...    n    n    y    n   \n",
       "2465  man week hellish. anyway time minut write assi...    n    y    n    n   \n",
       "2466  i gotten phone brady. im tri decid exacli week...    n    y    y    n   \n",
       "\n",
       "     cOPN  \n",
       "0       y  \n",
       "1       n  \n",
       "2       y  \n",
       "3       n  \n",
       "4       y  \n",
       "...   ...  \n",
       "2462    n  \n",
       "2463    y  \n",
       "2464    n  \n",
       "2465    y  \n",
       "2466    y  \n",
       "\n",
       "[2467 rows x 8 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_numbers(string):  #To remove numbers\n",
    "    return ''.join([i for i in string if not i.isdigit()])\n",
    "\n",
    "def remove_punctuations(string):  #To remove punctuations\n",
    "    punctuation = '''!()-[]{};:'\"\\,<>/?@#$%^&*_~'''\n",
    "    no_punctuation = \"\"\n",
    "    for char in string:\n",
    "       if char not in punctuation:\n",
    "           no_punctuation = no_punctuation + char\n",
    "    return no_punctuation\n",
    "def remove_stopwords(string):    #To remove the stopwords\n",
    "    sentence=[]\n",
    "    words=string.split()\n",
    "    for word in words:\n",
    "        if word not in stopwords.words():\n",
    "            sentence.append(word)\n",
    "    return ' '.join(sentence)\n",
    "def stemmer(sentence):           #To do the stemming using Porter stemmer\n",
    "    sent=[]\n",
    "    ps = PorterStemmer()\n",
    "    for i in sentence.split():\n",
    "        trial=ps.stem(i)\n",
    "        sent.append(trial)\n",
    "    return ' '.join(sent)\n",
    "            \n",
    "def text_preprocessing(df):      #This function takes all the preprocessing functions and modifies the input corpus\n",
    "    for i in tqdm(range(len(df))):\n",
    "        sen=df['TEXT'][i]\n",
    "        sen=remove_numbers(sen)\n",
    "        sen=remove_punctuations(sen)\n",
    "        sen=remove_stopwords(sen).lower()\n",
    "        sen=stemmer(sen)\n",
    "        df['TEXT'][i]=sen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d48d3da2fff43a8aa413cab3f472385",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2467.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>#AUTHID</th>\n",
       "      <th>TEXT</th>\n",
       "      <th>cEXT</th>\n",
       "      <th>cNEU</th>\n",
       "      <th>cAGR</th>\n",
       "      <th>cCON</th>\n",
       "      <th>cOPN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1997_504851.txt</td>\n",
       "      <td>well right i woke midday nap. it sort weird ev...</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "      <td>y</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1997_605191.txt</td>\n",
       "      <td>well go stream conscious essay. i use thing li...</td>\n",
       "      <td>n</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "      <td>n</td>\n",
       "      <td>n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1997_687252.txt</td>\n",
       "      <td>an open keyboard button push. the thing final ...</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "      <td>y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1997_568848.txt</td>\n",
       "      <td>i cant believ it realli happen my puls race li...</td>\n",
       "      <td>y</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "      <td>y</td>\n",
       "      <td>n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1997_688160.txt</td>\n",
       "      <td>well i go good old stream conscious assign aga...</td>\n",
       "      <td>y</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           #AUTHID                                               TEXT cEXT  \\\n",
       "0  1997_504851.txt  well right i woke midday nap. it sort weird ev...    n   \n",
       "1  1997_605191.txt  well go stream conscious essay. i use thing li...    n   \n",
       "2  1997_687252.txt  an open keyboard button push. the thing final ...    n   \n",
       "3  1997_568848.txt  i cant believ it realli happen my puls race li...    y   \n",
       "4  1997_688160.txt  well i go good old stream conscious assign aga...    y   \n",
       "\n",
       "  cNEU cAGR cCON cOPN  \n",
       "0    y    y    n    y  \n",
       "1    n    y    n    n  \n",
       "2    y    n    y    y  \n",
       "3    n    y    y    n  \n",
       "4    n    y    n    y  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_preprocessing(df) #Dataframe after thw text pre processing of the sentences\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('data_prep.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "306\n"
     ]
    }
   ],
   "source": [
    "# Maximum no. of sentences in a document\n",
    "max_c = 0\n",
    "for i in range(len(df['TEXT'])):\n",
    "    s = (df['TEXT'][i])\n",
    "    c = s.count('.')\n",
    "    if c>max_c:\n",
    "        max_c = c\n",
    "print(max_c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "vec = CountVectorizer()\n",
    "X = vec.fit_transform(df['TEXT'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_values"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(vec.vocabulary_.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'int'>\n"
     ]
    }
   ],
   "source": [
    "vocab = {}\n",
    "for i in vec.vocabulary_.keys():\n",
    "    vocab[i] = vec.vocabulary_[i].item()\n",
    "print(type(vocab['well']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'well': 21480,\n",
       " 'right': 16265,\n",
       " 'woke': 21839,\n",
       " 'midday': 12161,\n",
       " 'nap': 12810,\n",
       " 'it': 9998,\n",
       " 'sort': 18028,\n",
       " 'weird': 21456,\n",
       " 'ever': 6384,\n",
       " 'sinc': 17498,\n",
       " 'move': 12588,\n",
       " 'texa': 19337,\n",
       " 'problem': 15216,\n",
       " 'concentr': 3894,\n",
       " 'thing': 19534,\n",
       " 'rememb': 16013,\n",
       " 'start': 18364,\n",
       " 'homework': 8962,\n",
       " 'th': 19357,\n",
       " 'grade': 8020,\n",
       " 'soon': 17980,\n",
       " 'clock': 3561,\n",
       " 'struck': 18625,\n",
       " 'stop': 18519,\n",
       " 'done': 5455,\n",
       " 'of': 13446,\n",
       " 'cours': 4271,\n",
       " 'easier': 5798,\n",
       " 'still': 18475,\n",
       " 'but': 2684,\n",
       " 'got': 7988,\n",
       " 'littl': 11243,\n",
       " 'challeng': 3140,\n",
       " 'lot': 11412,\n",
       " 'busi': 2664,\n",
       " 'work': 21914,\n",
       " 'decid': 4793,\n",
       " 'spend': 18154,\n",
       " 'hour': 9075,\n",
       " 'get': 7732,\n",
       " 'alway': 691,\n",
       " 'paid': 14035,\n",
       " 'attent': 1341,\n",
       " 'class': 3467,\n",
       " 'plain': 14689,\n",
       " 'knew': 10660,\n",
       " 'stuff': 18667,\n",
       " 'look': 11368,\n",
       " 'back': 1487,\n",
       " 'realli': 15753,\n",
       " 'hard': 8456,\n",
       " 'stay': 18390,\n",
       " 'track': 20034,\n",
       " 'last': 10868,\n",
       " 'two': 20346,\n",
       " 'year': 22162,\n",
       " 'without': 21795,\n",
       " 'lazi': 10932,\n",
       " 'would': 21963,\n",
       " 'geniu': 7691,\n",
       " 'hey': 8743,\n",
       " 'that': 19377,\n",
       " 'good': 7947,\n",
       " 'late': 10875,\n",
       " 'correct': 4195,\n",
       " 'past': 14207,\n",
       " 'dont': 5466,\n",
       " 'know': 10675,\n",
       " 'focus': 7157,\n",
       " 'futur': 7555,\n",
       " 'the': 19401,\n",
       " 'peopl': 14369,\n",
       " 'say': 16759,\n",
       " 'bc': 1703,\n",
       " 'live': 11249,\n",
       " 'campu': 2833,\n",
       " 'cant': 2872,\n",
       " 'for': 7201,\n",
       " 'ala': 513,\n",
       " 'im': 9404,\n",
       " 'home': 8933,\n",
       " 'watch': 21350,\n",
       " 'eye': 6645,\n",
       " 'parent': 14142,\n",
       " 'nag': 12787,\n",
       " 'sister': 17532,\n",
       " 'you': 22239,\n",
       " 'point': 14798,\n",
       " 'anoth': 882,\n",
       " 'hassl': 8507,\n",
       " 'go': 7880,\n",
       " 'way': 21373,\n",
       " 'school': 16838,\n",
       " 'librari': 11100,\n",
       " 'studi': 18655,\n",
       " 'need': 12900,\n",
       " 'tell': 19254,\n",
       " 'wrong': 22044,\n",
       " 'see': 16982,\n",
       " 'theyr': 19499,\n",
       " 'come': 3745,\n",
       " 'away': 1433,\n",
       " 'theyv': 19501,\n",
       " 'shelter': 17259,\n",
       " 'much': 12623,\n",
       " 'worri': 21942,\n",
       " 'world': 21931,\n",
       " 'ask': 1215,\n",
       " 'keep': 10505,\n",
       " 'room': 16412,\n",
       " 'clean': 3499,\n",
       " 'help': 8670,\n",
       " 'even': 6373,\n",
       " 'enough': 6171,\n",
       " 'money': 12447,\n",
       " 'ut': 20878,\n",
       " 'dorm': 5506,\n",
       " 'apart': 990,\n",
       " 'next': 13008,\n",
       " 'semest': 17061,\n",
       " 'think': 19551,\n",
       " 'll': 11276,\n",
       " 'advantag': 306,\n",
       " 'topic': 19971,\n",
       " 'went': 21501,\n",
       " 'sixth': 17555,\n",
       " 'street': 18573,\n",
       " 'night': 13056,\n",
       " 'blast': 2101,\n",
       " 'havent': 8534,\n",
       " 'long': 11351,\n",
       " 'now': 13277,\n",
       " 'love': 11440,\n",
       " 'austin': 1383,\n",
       " 'when': 21567,\n",
       " 'va': 20913,\n",
       " 'use': 20859,\n",
       " 'dc': 4731,\n",
       " 'time': 19765,\n",
       " 'mani': 11688,\n",
       " 'student': 18653,\n",
       " 'run': 16546,\n",
       " 'around': 1148,\n",
       " 'fun': 7502,\n",
       " 'respons': 16122,\n",
       " 'abl': 64,\n",
       " 'prioriti': 15174,\n",
       " 'straight': 18538,\n",
       " 'question': 15537,\n",
       " 'wish': 21770,\n",
       " 'could': 4230,\n",
       " 'treat': 20111,\n",
       " 'like': 11153,\n",
       " 'person': 14455,\n",
       " 'screw': 16901,\n",
       " 'she': 17234,\n",
       " 'crazi': 4343,\n",
       " 'second': 16963,\n",
       " 'colleg': 3705,\n",
       " 'mess': 12073,\n",
       " 'whole': 21643,\n",
       " 'career': 2922,\n",
       " 'parti': 14174,\n",
       " 'and': 795,\n",
       " 'ultim': 20449,\n",
       " 'reason': 15789,\n",
       " 'anymor': 948,\n",
       " 'let': 11059,\n",
       " 'explor': 6598,\n",
       " 'indian': 9591,\n",
       " 'cultur': 4491,\n",
       " 'valu': 20940,\n",
       " 'they': 19493,\n",
       " 'mean': 11924,\n",
       " 'sens': 17085,\n",
       " 'meet': 11976,\n",
       " 'my': 12725,\n",
       " 'difficult': 5138,\n",
       " 'alreadi': 657,\n",
       " 'somehow': 17893,\n",
       " 'freedom': 7339,\n",
       " 'put': 15481,\n",
       " 'pressur': 15106,\n",
       " 'better': 1930,\n",
       " 'expect': 6561,\n",
       " 'write': 22022,\n",
       " 'anyth': 957,\n",
       " 'thought': 19632,\n",
       " 'order': 13744,\n",
       " 'so': 17802,\n",
       " 'hope': 9017,\n",
       " 'read': 15724,\n",
       " 'luck': 11487,\n",
       " 'ta': 19046,\n",
       " 'stream': 18565,\n",
       " 'conscious': 4010,\n",
       " 'essay': 6314,\n",
       " 'high': 8779,\n",
       " 'sometim': 17931,\n",
       " 'pretti': 15125,\n",
       " 'interest': 9805,\n",
       " 'often': 13475,\n",
       " 'find': 6969,\n",
       " 'lack': 10776,\n",
       " 'normal': 13201,\n",
       " 'consid': 4030,\n",
       " 'someon': 17904,\n",
       " 'wonder': 21858,\n",
       " 'hit': 8856,\n",
       " 'enter': 6186,\n",
       " 'send': 17081,\n",
       " 'front': 7442,\n",
       " 'mayb': 11870,\n",
       " 'ill': 9383,\n",
       " 'fix': 7034,\n",
       " 'later': 10877,\n",
       " 'friend': 7393,\n",
       " 'play': 14718,\n",
       " 'guitar': 8235,\n",
       " 'anyway': 967,\n",
       " 'more': 12510,\n",
       " 'he': 8571,\n",
       " 'learn': 10963,\n",
       " 'there': 19458,\n",
       " 'draw': 5582,\n",
       " 'wall': 21273,\n",
       " 'comic': 3761,\n",
       " 'book': 2244,\n",
       " 'charact': 3168,\n",
       " 'sure': 18889,\n",
       " 'ive': 10048,\n",
       " 'kept': 10535,\n",
       " 'heard': 8594,\n",
       " 'sound': 18050,\n",
       " 'icq': 9316,\n",
       " 'chat': 3193,\n",
       " 'program': 15268,\n",
       " 'internet': 9825,\n",
       " 'explain': 6590,\n",
       " 'nice': 13021,\n",
       " 'talk': 19105,\n",
       " 'weekend': 21434,\n",
       " 'labor': 10765,\n",
       " 'day': 4708,\n",
       " 'brother': 2515,\n",
       " 'didnt': 5106,\n",
       " 'no': 13118,\n",
       " 'guess': 8217,\n",
       " 'hmm': 8871,\n",
       " 'wow': 21978,\n",
       " 'line': 11191,\n",
       " 'wont': 21874,\n",
       " 'chang': 3151,\n",
       " 'okay': 13527,\n",
       " 'found': 7276,\n",
       " 'happen': 8441,\n",
       " 'convers': 4110,\n",
       " 'not': 13227,\n",
       " 'someth': 17913,\n",
       " 'might': 12184,\n",
       " 'phone': 14555,\n",
       " 'rang': 15662,\n",
       " 'should': 17359,\n",
       " 'guy': 8270,\n",
       " 'answer': 889,\n",
       " 'roommat': 16418,\n",
       " 'suitem': 18787,\n",
       " 'came': 2818,\n",
       " 'uncomfort': 20523,\n",
       " 'bathroom': 1675,\n",
       " 'bore': 2277,\n",
       " 'piec': 14603,\n",
       " 'literatur': 11235,\n",
       " 'never': 12980,\n",
       " 'realiz': 15751,\n",
       " 'dull': 5692,\n",
       " 'everyday': 6405,\n",
       " 'then': 19440,\n",
       " 'mind': 12229,\n",
       " 'constantli': 4047,\n",
       " 'isnt': 9981,\n",
       " 'deepli': 4827,\n",
       " 'ten': 19275,\n",
       " 'minut': 12270,\n",
       " 'onli': 13623,\n",
       " 'how': 9104,\n",
       " 'sad': 16605,\n",
       " 'rest': 16127,\n",
       " 'thi': 19504,\n",
       " 'seat': 16950,\n",
       " 'hurt': 9222,\n",
       " 'arthriti': 1173,\n",
       " 'older': 13542,\n",
       " 'wouldnt': 21970,\n",
       " 'grow': 8176,\n",
       " 'old': 13541,\n",
       " 'suppos': 18873,\n",
       " 'activ': 213,\n",
       " 'fear': 6821,\n",
       " 'itll': 10024,\n",
       " 'age': 397,\n",
       " 'gradual': 8027,\n",
       " 'id': 9318,\n",
       " 'deal': 4755,\n",
       " 'paralysi': 14129,\n",
       " 'accid': 140,\n",
       " 'though': 19625,\n",
       " 'as': 1190,\n",
       " 'god': 7888,\n",
       " 'thirsti': 19577,\n",
       " 'drink': 5619,\n",
       " 'frisbe': 7422,\n",
       " 'summer': 18802,\n",
       " 'game': 7597,\n",
       " 'tire': 19802,\n",
       " 'shape': 17215,\n",
       " 'hate': 8513,\n",
       " 'hmmm': 8873,\n",
       " 'almost': 633,\n",
       " 'just': 10405,\n",
       " 'make': 11638,\n",
       " 'short': 17336,\n",
       " 'reachabl': 15720,\n",
       " 'goal': 7881,\n",
       " 'whatev': 21543,\n",
       " 'els': 6031,\n",
       " 'tonight': 19935,\n",
       " 'shirt': 17299,\n",
       " 'smell': 17721,\n",
       " 'dinner': 5184,\n",
       " 'disgust': 5273,\n",
       " 'wake': 21260,\n",
       " 'tomorrow': 19916,\n",
       " 'wasnt': 21342,\n",
       " 'earli': 5781,\n",
       " 'made': 11587,\n",
       " 'proud': 15334,\n",
       " 'sarcasm': 16714,\n",
       " 'narr': 12818,\n",
       " 'oh': 13493,\n",
       " 'head': 8572,\n",
       " 'until': 20769,\n",
       " 'bye': 2724,\n",
       " 'an': 769,\n",
       " 'open': 13682,\n",
       " 'keyboard': 10554,\n",
       " 'button': 2703,\n",
       " 'push': 15474,\n",
       " 'final': 6963,\n",
       " 'period': 14425,\n",
       " 'comma': 3770,\n",
       " 'doubl': 5528,\n",
       " 'space': 18078,\n",
       " 'we': 21384,\n",
       " 'word': 21905,\n",
       " 'happi': 8445,\n",
       " 'assign': 1243,\n",
       " 'jibberjabb': 10246,\n",
       " 'babbl': 1474,\n",
       " 'relax': 15969,\n",
       " 'feel': 6848,\n",
       " 'silli': 17467,\n",
       " 'grammat': 8038,\n",
       " 'incorrect': 9563,\n",
       " 'link': 11204,\n",
       " 'unknown': 20678,\n",
       " 'reader': 15726,\n",
       " 'graduat': 8028,\n",
       " 'absurd': 113,\n",
       " 'job': 10275,\n",
       " 'type': 20369,\n",
       " 'jabber': 10074,\n",
       " 'dinoflagel': 5188,\n",
       " 'about': 86,\n",
       " 'sunflow': 18817,\n",
       " 'cross': 4409,\n",
       " 'fiberglass': 6919,\n",
       " 'ad': 233,\n",
       " 'lip': 11213,\n",
       " 'via': 21068,\n",
       " 'clove': 3588,\n",
       " 'cigarett': 3403,\n",
       " 'shouldnt': 17365,\n",
       " 'solidifi': 17873,\n",
       " 'lodg': 11312,\n",
       " 'behind': 1834,\n",
       " 'tongu': 19932,\n",
       " 'somewher': 17945,\n",
       " 'hide': 8773,\n",
       " 'dinoflaghel': 5189,\n",
       " 'protista': 15329,\n",
       " 'weed': 21419,\n",
       " 'san': 16674,\n",
       " 'luiz': 11509,\n",
       " 'valley': 20939,\n",
       " 'mushroom': 12690,\n",
       " 'cow': 4288,\n",
       " 'shit': 17301,\n",
       " 'tico': 19731,\n",
       " 'chigger': 3274,\n",
       " 'leg': 11002,\n",
       " 'itch': 10007,\n",
       " 'coat': 3631,\n",
       " 'clear': 3507,\n",
       " 'nail': 12791,\n",
       " 'polish': 14821,\n",
       " 'suffoc': 18776,\n",
       " 'bug': 2586,\n",
       " 'marco': 11725,\n",
       " 'ecuador': 5864,\n",
       " 'sentenc': 17100,\n",
       " 'affect': 340,\n",
       " 'slowish': 17683,\n",
       " 'speed': 18139,\n",
       " 'measur': 11942,\n",
       " 'gener': 7682,\n",
       " 'dorki': 5505,\n",
       " 'psyc': 15375,\n",
       " 'green': 8104,\n",
       " 'tabl': 19051,\n",
       " 'kitchen': 10633,\n",
       " 'vomit': 21198,\n",
       " 'orang': 13736,\n",
       " 'what': 21538,\n",
       " 'color': 3726,\n",
       " 'unit': 20664,\n",
       " 'state': 18375,\n",
       " 'greencard': 8106,\n",
       " 'dr': 5565,\n",
       " 'linder': 11188,\n",
       " 'lab': 10762,\n",
       " 'fund': 7506,\n",
       " 'pay': 14258,\n",
       " 'may': 11868,\n",
       " 'brainless': 2369,\n",
       " 'nothing': 13246,\n",
       " 'wallow': 21278,\n",
       " 'boredom': 2279,\n",
       " 'entir': 6198,\n",
       " 'bad': 1518,\n",
       " 'focu': 7156,\n",
       " 'organ': 13756,\n",
       " 'mistak': 12330,\n",
       " 'eat': 5813,\n",
       " 'hungri': 9206,\n",
       " 'food': 7178,\n",
       " 'comput': 3872,\n",
       " 'rambl': 15638,\n",
       " 'edit': 5877,\n",
       " 'hannah': 8432,\n",
       " 'imi': 9426,\n",
       " 'osdprey': 13787,\n",
       " 'hous': 9081,\n",
       " 'spider': 18165,\n",
       " 'pain': 14038,\n",
       " 'place': 14681,\n",
       " 'paint': 14041,\n",
       " 'cloud': 3585,\n",
       " 'ceil': 3074,\n",
       " 'blue': 2155,\n",
       " 'obnoxi': 13377,\n",
       " 'carey': 2930,\n",
       " 'sex': 17170,\n",
       " 'yeah': 22157,\n",
       " 'strang': 18550,\n",
       " 'portonoy': 14909,\n",
       " 'complaint': 3839,\n",
       " 'ring': 16285,\n",
       " 'eager': 5767,\n",
       " 'finish': 6983,\n",
       " 'whom': 21648,\n",
       " 'bell': 1861,\n",
       " 'toll': 19896,\n",
       " 'bio': 2015,\n",
       " 'carbon': 2911,\n",
       " 'atom': 1323,\n",
       " 'bond': 2227,\n",
       " 'orbit': 13738,\n",
       " 'electron': 5991,\n",
       " 'configur': 3949,\n",
       " 'surround': 18916,\n",
       " 'letter': 11066,\n",
       " 'first': 7014,\n",
       " 'must': 12700,\n",
       " 'grant': 8062,\n",
       " 'full': 7491,\n",
       " 'spin': 18172,\n",
       " 'opposit': 13709,\n",
       " 'direct': 5205,\n",
       " 'scottish': 16880,\n",
       " 'poetri': 14794,\n",
       " 'mike': 12195,\n",
       " 'kilt': 10595,\n",
       " 'slowli': 17685,\n",
       " 'goe': 7904,\n",
       " 'entertain': 6190,\n",
       " 'grad': 8019,\n",
       " 'scan': 16778,\n",
       " 'random': 15657,\n",
       " 'me': 11916,\n",
       " 'botani': 2295,\n",
       " 'il': 9378,\n",
       " 'belli': 1862,\n",
       " 'flamenco': 7053,\n",
       " 'bjork': 2061,\n",
       " 'rozamond': 16484,\n",
       " 'cockril': 3643,\n",
       " 'kickin': 10575,\n",
       " 'saratoga': 16713,\n",
       " 'spring': 18240,\n",
       " 'molli': 12422,\n",
       " 'exchang': 6509,\n",
       " 'worthwhil': 21956,\n",
       " 'ferret': 6885,\n",
       " 'manag': 11670,\n",
       " 'wast': 21347,\n",
       " 'who': 21635,\n",
       " 'if': 9342,\n",
       " 'took': 19951,\n",
       " 'noth': 13242,\n",
       " 'financi': 6968,\n",
       " 'aid': 475,\n",
       " 'sprinkl': 18245,\n",
       " 'sign': 17451,\n",
       " 'outta': 13897,\n",
       " 'tree': 20117,\n",
       " 'believ': 1857,\n",
       " 'puls': 15426,\n",
       " 'race': 15580,\n",
       " 'mad': 11584,\n",
       " 'step': 18421,\n",
       " 'sleep': 17631,\n",
       " 'million': 12213,\n",
       " 'deadlin': 4749,\n",
       " 'excit': 6510,\n",
       " 'ye': 22153,\n",
       " 'wait': 21255,\n",
       " 'calm': 2808,\n",
       " 'arent': 1100,\n",
       " 'offici': 13460,\n",
       " 'coupl': 4265,\n",
       " 'yet': 22202,\n",
       " 'horribl': 9040,\n",
       " 'great': 8089,\n",
       " 'calculu': 2779,\n",
       " 'ami': 729,\n",
       " 'bob': 2191,\n",
       " 'miracl': 12287,\n",
       " 'worker': 21918,\n",
       " 'turn': 20289,\n",
       " 'owe': 13989,\n",
       " 'repay': 16058,\n",
       " 'steve': 18450,\n",
       " 'malaysia': 11651,\n",
       " 'month': 12480,\n",
       " 'refriger': 15903,\n",
       " 'omigosh': 13572,\n",
       " 'lunch': 11518,\n",
       " 'perfect': 14414,\n",
       " 'everi': 6393,\n",
       " 'imagin': 9410,\n",
       " 'import': 9476,\n",
       " 'certainli': 3113,\n",
       " 'dynam': 5757,\n",
       " 'inde': 9577,\n",
       " 'poster': 14932,\n",
       " 'morn': 12523,\n",
       " 'alarm': 524,\n",
       " 'reliabl': 15981,\n",
       " 'oversleep': 13968,\n",
       " 'tragic': 20050,\n",
       " 'slept': 17644,\n",
       " 'noon': 13185,\n",
       " 'miss': 12316,\n",
       " 'thank': 19369,\n",
       " 'portia': 14902,\n",
       " 'along': 640,\n",
       " 'definit': 4860,\n",
       " 'support': 18872,\n",
       " 'tension': 19291,\n",
       " 'eas': 5796,\n",
       " 'rehears': 15935,\n",
       " 'beforehand': 1814,\n",
       " 'nervou': 12954,\n",
       " 'ok': 13525,\n",
       " 'whi': 21587,\n",
       " 'human': 9173,\n",
       " 'tortur': 19986,\n",
       " 'worth': 21952,\n",
       " 'freshman': 7370,\n",
       " 'hs': 9126,\n",
       " 'english': 6142,\n",
       " 'teacher': 19200,\n",
       " 'us': 20849,\n",
       " 'complet': 3842,\n",
       " 'ridicul': 16251,\n",
       " 'taco': 19060,\n",
       " 'except': 6501,\n",
       " 'in': 9504,\n",
       " 'psycholog': 15391,\n",
       " 'free': 7336,\n",
       " 'big': 1978,\n",
       " 'figur': 6943,\n",
       " 'screwup': 16906,\n",
       " 'true': 20205,\n",
       " 'yall': 22131,\n",
       " 'idea': 9321,\n",
       " 'today': 19857,\n",
       " 'nut': 13323,\n",
       " 'case': 2989,\n",
       " 'kind': 10603,\n",
       " 'gal': 7581,\n",
       " 'actual': 219,\n",
       " 'biscuit': 2037,\n",
       " 'gravi': 8080,\n",
       " 'breakfast': 2413,\n",
       " 'countri': 4259,\n",
       " 'fri': 7377,\n",
       " 'steak': 18398,\n",
       " 'okra': 13538,\n",
       " 'connecticut': 3981,\n",
       " 'have': 8527,\n",
       " 'less': 11049,\n",
       " 'worship': 21949,\n",
       " 'veget': 20989,\n",
       " 'goddess': 7894,\n",
       " 'blankperform': 2096,\n",
       " 'spot': 18221,\n",
       " 'herei': 8706,\n",
       " 'your': 22252,\n",
       " 'stress': 18587,\n",
       " 'probabl': 15203,\n",
       " 'listen': 11225,\n",
       " 'incoher': 9543,\n",
       " 'paragraph': 14126,\n",
       " 'these': 19485,\n",
       " 'ass': 1226,\n",
       " 'here': 8704,\n",
       " 'smf': 17729,\n",
       " 'bajillion': 1550,\n",
       " 'unfortun': 20631,\n",
       " 'weve': 21526,\n",
       " 'tri': 20133,\n",
       " 'quit': 15562,\n",
       " 'center': 3095,\n",
       " 'yippe': 22214,\n",
       " 'aboutl': 92,\n",
       " 'at': 1292,\n",
       " 'thingi': 19538,\n",
       " 'blah': 2080,\n",
       " 'dedic': 4813,\n",
       " 'procrastin': 15234,\n",
       " 'middl': 12162,\n",
       " 'by': 2722,\n",
       " 'damn': 4620,\n",
       " 'harder': 8462,\n",
       " 'skill': 17581,\n",
       " 'trip': 20161,\n",
       " 'couldnt': 4234,\n",
       " 'town': 20019,\n",
       " 'everyth': 6423,\n",
       " 'famili': 6717,\n",
       " 'real': 15743,\n",
       " 'quick': 15544,\n",
       " 'wors': 21946,\n",
       " 'least': 10972,\n",
       " 'boyfriend': 2337,\n",
       " 'crap': 4328,\n",
       " 'car': 2903,\n",
       " 'after': 370,\n",
       " 'life': 11115,\n",
       " 'readi': 15728,\n",
       " 'seem': 16990,\n",
       " 'citi': 3426,\n",
       " 'enjoy': 6159,\n",
       " 'four': 7282,\n",
       " 'while': 21595,\n",
       " 'subject': 18712,\n",
       " 'professor': 15251,\n",
       " 'administr': 276,\n",
       " 'give': 7823,\n",
       " 'speech': 18137,\n",
       " 'sorri': 18025,\n",
       " 'plan': 14691,\n",
       " 'offens': 13455,\n",
       " 'undergrad': 20564,\n",
       " 'knock': 10669,\n",
       " 'exactli': 6476,\n",
       " 'nifti': 13050,\n",
       " 'fulfil': 7490,\n",
       " 'analyz': 781,\n",
       " 'regard': 15910,\n",
       " 'mysteri': 12762,\n",
       " 'illog': 9390,\n",
       " 'connect': 3979,\n",
       " 'pattern': 14243,\n",
       " 'suggest': 18780,\n",
       " 'box': 2324,\n",
       " 'scroll': 16913,\n",
       " 'across': 205,\n",
       " 'unless': 20682,\n",
       " 'take': 19087,\n",
       " 'opportun': 13705,\n",
       " 'easi': 5797,\n",
       " 'had': 8310,\n",
       " 'music': 12692,\n",
       " 'kvrx': 10753,\n",
       " 'radio': 15598,\n",
       " 'show': 17373,\n",
       " 'don': 5452,\n",
       " 'depth': 4968,\n",
       " 'jeez': 10185,\n",
       " 'song': 17966,\n",
       " 'quietli': 15553,\n",
       " 'cool': 4133,\n",
       " 'begin': 1825,\n",
       " 'louder': 11424,\n",
       " 'rather': 15691,\n",
       " 'erot': 6271,\n",
       " 'deep': 4823,\n",
       " 'disturb': 5348,\n",
       " 'whether': 21581,\n",
       " 'medic': 11960,\n",
       " 'technolog': 19220,\n",
       " 'form': 7245,\n",
       " 'chemistri': 3242,\n",
       " 'mycolog': 12729,\n",
       " 'religi': 15987,\n",
       " 'speak': 18117,\n",
       " 'psa': 15363,\n",
       " 'pagan': 14030,\n",
       " 'allianc': 598,\n",
       " 'ahh': 455,\n",
       " 'gotta': 7995,\n",
       " 'screech': 16894,\n",
       " 'chair': 3133,\n",
       " 'ahhhhh': 459,\n",
       " 'sever': 17162,\n",
       " 'caleb': 2782,\n",
       " 'women': 21851,\n",
       " 'bound': 2313,\n",
       " 'wise': 21768,\n",
       " 'unabl': 20478,\n",
       " 'charg': 3173,\n",
       " 'booth': 2263,\n",
       " 'monday': 12440,\n",
       " 'tuesday': 20257,\n",
       " 'team': 19206,\n",
       " 'win': 21731,\n",
       " 'yeesh': 22176,\n",
       " 'poor': 14864,\n",
       " 'lalalalalalal': 10806,\n",
       " 'shaman': 17206,\n",
       " 'appar': 1010,\n",
       " 'siberian': 17416,\n",
       " 'hundr': 9198,\n",
       " 'differ': 5126,\n",
       " 'follow': 7171,\n",
       " 'due': 5684,\n",
       " 'vast': 20968,\n",
       " 'number': 13311,\n",
       " 'tribe': 20138,\n",
       " 'speckl': 18132,\n",
       " 'peter': 14489,\n",
       " 'steel': 18404,\n",
       " 'sexi': 17173,\n",
       " 'voic': 21178,\n",
       " 'info': 9658,\n",
       " 'dto': 5673,\n",
       " 'ellips': 6025,\n",
       " 'perhap': 14421,\n",
       " 'remind': 16021,\n",
       " 'young': 22247,\n",
       " 'ride': 16247,\n",
       " 'streetlight': 18576,\n",
       " 'bright': 2457,\n",
       " 'either': 5960,\n",
       " 'epilepsi': 6226,\n",
       " 'taken': 19089,\n",
       " 'incarn': 9522,\n",
       " 'doe': 5405,\n",
       " 'possibl': 14927,\n",
       " 'clairvoy': 3444,\n",
       " 'assumpt': 1266,\n",
       " 'spirit': 18178,\n",
       " 'equal': 6241,\n",
       " 'shall': 17202,\n",
       " 'childhood': 3280,\n",
       " 'bear': 1724,\n",
       " 'mark': 11754,\n",
       " 'fuzzi': 7559,\n",
       " 'recollect': 15837,\n",
       " 'exact': 6475,\n",
       " 'detail': 5019,\n",
       " 'other': 13807,\n",
       " 'brown': 2524,\n",
       " 'best': 1915,\n",
       " 'suck': 18758,\n",
       " 'cd': 3061,\n",
       " 'lie': 11107,\n",
       " 'machin': 11575,\n",
       " 'circl': 3416,\n",
       " 'psychot': 15397,\n",
       " 'manner': 11702,\n",
       " 'hypnot': 9272,\n",
       " 'sedat': 16980,\n",
       " 'occas': 13401,\n",
       " 'grey': 8123,\n",
       " 'neutral': 12977,\n",
       " 'passiv': 14200,\n",
       " 'taoism': 19144,\n",
       " 'philosophi': 14543,\n",
       " 'current': 4518,\n",
       " 'aleist': 549,\n",
       " 'crowley': 4424,\n",
       " 'hi': 8757,\n",
       " 'hermet': 8717,\n",
       " 'bit': 2041,\n",
       " 'abnorm': 70,\n",
       " 'eccentr': 5834,\n",
       " 'side': 17428,\n",
       " 'stori': 18529,\n",
       " 'milkmen': 12209,\n",
       " 'field': 6927,\n",
       " 'rose': 16438,\n",
       " 'daisi': 4594,\n",
       " 'upon': 20818,\n",
       " 'regress': 15923,\n",
       " 'anyon': 951,\n",
       " 'famou': 6727,\n",
       " 'popular': 14884,\n",
       " 'hot': 9060,\n",
       " 'left': 10998,\n",
       " 'finger': 6978,\n",
       " 'press': 15105,\n",
       " 'heavi': 8616,\n",
       " 'fast': 6764,\n",
       " 'lith': 11236,\n",
       " 'light': 11137,\n",
       " 'record': 15846,\n",
       " 'said': 16626,\n",
       " 'three': 19656,\n",
       " 'pass': 14194,\n",
       " 'slow': 17680,\n",
       " 'halt': 8383,\n",
       " 'am': 699,\n",
       " 'intens': 9796,\n",
       " 'howev': 9108,\n",
       " 'blank': 2093,\n",
       " 'econ': 5854,\n",
       " 'quiz': 15567,\n",
       " 'chanc': 3148,\n",
       " 'faith': 6701,\n",
       " 'motto': 12569,\n",
       " 'funni': 7522,\n",
       " 'one': 13591,\n",
       " 'gone': 7936,\n",
       " 'louie': 11429,\n",
       " 'lug': 11504,\n",
       " 'mcgurg': 11901,\n",
       " 'exampl': 6484,\n",
       " 'die': 5110,\n",
       " 'somebodi': 17888,\n",
       " 'kill': 10590,\n",
       " 'hang': 8421,\n",
       " 'mr': 12607,\n",
       " 'sell': 17056,\n",
       " 'appl': 1021,\n",
       " 'ahead': 454,\n",
       " 'govern': 8004,\n",
       " 'oscar': 13786,\n",
       " 'movi': 12594,\n",
       " 'wen': 21494,\n",
       " 'conscienc': 4003,\n",
       " 'paper': 14104,\n",
       " 'submit': 18719,\n",
       " 'longer': 11353,\n",
       " 'everyon': 6416,\n",
       " 'sit': 17540,\n",
       " 'usual': 20875,\n",
       " 'cricket': 4379,\n",
       " 'sport': 18215,\n",
       " 'sahara': 16625,\n",
       " 'cup': 4501,\n",
       " 'canada': 2839,\n",
       " 'india': 9590,\n",
       " 'vc': 20976,\n",
       " 'paistan': 14045,\n",
       " 'biggest': 1981,\n",
       " 'rivalri': 16312,\n",
       " 'pak': 14047,\n",
       " 'victori': 21088,\n",
       " 'aamir': 22,\n",
       " 'sohai': 17855,\n",
       " 'drop': 5639,\n",
       " 'bowler': 2323,\n",
       " 'park': 14156,\n",
       " 'bodi': 2199,\n",
       " 'doubt': 5533,\n",
       " 'whoever': 21639,\n",
       " 'garbag': 7612,\n",
       " 'screen': 16895,\n",
       " 'pop': 14869,\n",
       " 'is': 9957,\n",
       " 'ploy': 14758,\n",
       " 'written': 22038,\n",
       " 'fact': 6679,\n",
       " 'notic': 13250,\n",
       " 'veri': 21031,\n",
       " 'tick': 19724,\n",
       " 'gain': 7580,\n",
       " 'devic': 5043,\n",
       " 'told': 19892,\n",
       " 'scari': 16789,\n",
       " 'nowaday': 13278,\n",
       " 'proofread': 15298,\n",
       " 'expert': 6585,\n",
       " 'typer': 20371,\n",
       " 'matter': 11848,\n",
       " 'amaz': 707,\n",
       " 'all': 581,\n",
       " 'rtf': 16509,\n",
       " 'usenet': 20863,\n",
       " 'site': 17543,\n",
       " 'sent': 17097,\n",
       " 'requir': 16090,\n",
       " 'messag': 12074,\n",
       " 'from': 7434,\n",
       " 'slot': 17675,\n",
       " 'credit': 4367,\n",
       " 'access': 138,\n",
       " 'or': 13732,\n",
       " 'bring': 2468,\n",
       " 'basic': 1649,\n",
       " 'desk': 4996,\n",
       " 'antiqu': 917,\n",
       " 'spent': 18155,\n",
       " 'favorit': 6800,\n",
       " 'hear': 8593,\n",
       " 'journey': 10333,\n",
       " 'string': 18611,\n",
       " 'stereo': 18439,\n",
       " 'hall': 8374,\n",
       " 'shut': 17405,\n",
       " 'mile': 12202,\n",
       " 'earn': 5786,\n",
       " 'squander': 18258,\n",
       " 'stupid': 18686,\n",
       " 'effect': 5911,\n",
       " 'loud': 11423,\n",
       " 'want': 21300,\n",
       " 'inspector': 9745,\n",
       " 'gadget': 7574,\n",
       " 'simpson': 17492,\n",
       " 'channel': 3156,\n",
       " 'choos': 3341,\n",
       " 'inconsider': 9554,\n",
       " 'shmuck': 17311,\n",
       " 'tight': 19749,\n",
       " 'plush': 14771,\n",
       " 'carpet': 2959,\n",
       " 'feet': 6859,\n",
       " 'cold': 3680,\n",
       " 'refresh': 15901,\n",
       " 'tast': 19168,\n",
       " 'coke': 3677,\n",
       " 'walk': 21267,\n",
       " 'quickli': 15547,\n",
       " 'countless': 4258,\n",
       " 'mindless': 12234,\n",
       " 'televis': 19252,\n",
       " 'pure': 15456,\n",
       " 'mindnumb': 12236,\n",
       " 'beat': 1730,\n",
       " 'industri': 9610,\n",
       " 'bed': 1786,\n",
       " 'jake': 10096,\n",
       " 'gave': 7644,\n",
       " 'nip': 13091,\n",
       " 'scratch': 16890,\n",
       " 'ear': 5777,\n",
       " 'bumpkin': 2629,\n",
       " 'extermin': 6620,\n",
       " 'larg': 10854,\n",
       " 'invit': 9900,\n",
       " 'vet': 21057,\n",
       " 'call': 2793,\n",
       " 'bigbon': 1979,\n",
       " 'doesn': 5411,\n",
       " 'obes': 13364,\n",
       " 'ate': 1296,\n",
       " 'bowl': 2322,\n",
       " 'pound': 14953,\n",
       " 'gut': 8263,\n",
       " 'cat': 3013,\n",
       " 'twenti': 20320,\n",
       " 'pounder': 14955,\n",
       " 'equival': 6249,\n",
       " 'theatr': 19406,\n",
       " 'rock': 16348,\n",
       " ...}"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('vocab_data.json', 'w') as fp:\n",
    "    json.dump(vocab, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(df['TEXT'])):\n",
    "    file = open('./texts/'+str(i)+'.txt',mode='w')\n",
    "    file.write(df['TEXT'][i])\n",
    "    file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = arff.load(open('mairesse_feat_m5reg.arff','r'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = pd.read_csv('mairesse_feat_m5reg.csv',encoding = \"ISO-8859-1\") #training_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>filename</th>\n",
       "      <th>AOA</th>\n",
       "      <th>BROWN-FREQ</th>\n",
       "      <th>CONC</th>\n",
       "      <th>FAM</th>\n",
       "      <th>IMAG</th>\n",
       "      <th>K-F-FREQ</th>\n",
       "      <th>K-F-NCATS</th>\n",
       "      <th>K-F-NSAMP</th>\n",
       "      <th>...</th>\n",
       "      <th>Quote</th>\n",
       "      <th>Apostro</th>\n",
       "      <th>Parenth</th>\n",
       "      <th>OtherP</th>\n",
       "      <th>AllPct</th>\n",
       "      <th>Extraversion</th>\n",
       "      <th>'Emotional</th>\n",
       "      <th>Agreeableness</th>\n",
       "      <th>Conscientiousness</th>\n",
       "      <th>'Openness</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>/home/shubham/PersonalityRecognizer/texts/910.txt</td>\n",
       "      <td>1.364566</td>\n",
       "      <td>?</td>\n",
       "      <td>-0.977067</td>\n",
       "      <td>0.768194</td>\n",
       "      <td>-0.562073</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.193295</td>\n",
       "      <td>-0.610687</td>\n",
       "      <td>4.796426</td>\n",
       "      <td>4.213128</td>\n",
       "      <td>4.492554</td>\n",
       "      <td>4.332998</td>\n",
       "      <td>4.903835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>/home/shubham/PersonalityRecognizer/texts/1821...</td>\n",
       "      <td>-0.640854</td>\n",
       "      <td>?</td>\n",
       "      <td>-0.189962</td>\n",
       "      <td>-0.257596</td>\n",
       "      <td>0.056930</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.193295</td>\n",
       "      <td>0.283737</td>\n",
       "      <td>4.370836</td>\n",
       "      <td>4.052020</td>\n",
       "      <td>4.284788</td>\n",
       "      <td>4.893034</td>\n",
       "      <td>4.973424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>/home/shubham/PersonalityRecognizer/texts/1679...</td>\n",
       "      <td>-1.072180</td>\n",
       "      <td>?</td>\n",
       "      <td>0.071708</td>\n",
       "      <td>-1.856507</td>\n",
       "      <td>1.156928</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.193295</td>\n",
       "      <td>-2.628056</td>\n",
       "      <td>4.678106</td>\n",
       "      <td>3.585848</td>\n",
       "      <td>4.284788</td>\n",
       "      <td>4.893034</td>\n",
       "      <td>4.973424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>/home/shubham/PersonalityRecognizer/texts/2461...</td>\n",
       "      <td>-0.659012</td>\n",
       "      <td>?</td>\n",
       "      <td>1.315691</td>\n",
       "      <td>-0.945107</td>\n",
       "      <td>1.666795</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.521484</td>\n",
       "      <td>-0.669396</td>\n",
       "      <td>4.945070</td>\n",
       "      <td>3.585848</td>\n",
       "      <td>4.492554</td>\n",
       "      <td>4.819516</td>\n",
       "      <td>4.973424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>/home/shubham/PersonalityRecognizer/texts/748.txt</td>\n",
       "      <td>-1.319069</td>\n",
       "      <td>?</td>\n",
       "      <td>0.795813</td>\n",
       "      <td>-0.273787</td>\n",
       "      <td>1.163599</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.193295</td>\n",
       "      <td>0.209775</td>\n",
       "      <td>4.370836</td>\n",
       "      <td>3.767593</td>\n",
       "      <td>4.492554</td>\n",
       "      <td>4.472535</td>\n",
       "      <td>4.253278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2462</th>\n",
       "      <td>2463</td>\n",
       "      <td>/home/shubham/PersonalityRecognizer/texts/1717...</td>\n",
       "      <td>0.259374</td>\n",
       "      <td>?</td>\n",
       "      <td>1.088820</td>\n",
       "      <td>-2.174183</td>\n",
       "      <td>1.571568</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.193295</td>\n",
       "      <td>-0.077060</td>\n",
       "      <td>4.945070</td>\n",
       "      <td>3.585848</td>\n",
       "      <td>4.711398</td>\n",
       "      <td>4.497862</td>\n",
       "      <td>4.973424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2463</th>\n",
       "      <td>2464</td>\n",
       "      <td>/home/shubham/PersonalityRecognizer/texts/52.txt</td>\n",
       "      <td>-0.772750</td>\n",
       "      <td>?</td>\n",
       "      <td>0.432476</td>\n",
       "      <td>0.334986</td>\n",
       "      <td>0.215733</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.193295</td>\n",
       "      <td>0.068354</td>\n",
       "      <td>4.678106</td>\n",
       "      <td>4.213128</td>\n",
       "      <td>4.655560</td>\n",
       "      <td>4.893034</td>\n",
       "      <td>4.681388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2464</th>\n",
       "      <td>2465</td>\n",
       "      <td>/home/shubham/PersonalityRecognizer/texts/1978...</td>\n",
       "      <td>-0.294362</td>\n",
       "      <td>?</td>\n",
       "      <td>0.434861</td>\n",
       "      <td>0.047322</td>\n",
       "      <td>0.704492</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.193295</td>\n",
       "      <td>-1.619049</td>\n",
       "      <td>4.945070</td>\n",
       "      <td>3.767593</td>\n",
       "      <td>4.655560</td>\n",
       "      <td>4.893034</td>\n",
       "      <td>4.681388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2465</th>\n",
       "      <td>2466</td>\n",
       "      <td>/home/shubham/PersonalityRecognizer/texts/363.txt</td>\n",
       "      <td>-0.193252</td>\n",
       "      <td>?</td>\n",
       "      <td>0.285823</td>\n",
       "      <td>-0.405907</td>\n",
       "      <td>0.815238</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.193295</td>\n",
       "      <td>-0.775789</td>\n",
       "      <td>4.646799</td>\n",
       "      <td>3.584963</td>\n",
       "      <td>4.653380</td>\n",
       "      <td>4.893034</td>\n",
       "      <td>4.583562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2466</th>\n",
       "      <td>2467</td>\n",
       "      <td>/home/shubham/PersonalityRecognizer/texts/320.txt</td>\n",
       "      <td>0.536508</td>\n",
       "      <td>?</td>\n",
       "      <td>-0.135137</td>\n",
       "      <td>-0.531425</td>\n",
       "      <td>-0.539247</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.184312</td>\n",
       "      <td>0.552863</td>\n",
       "      <td>4.710589</td>\n",
       "      <td>3.767593</td>\n",
       "      <td>4.534645</td>\n",
       "      <td>4.792520</td>\n",
       "      <td>4.857327</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2467 rows × 91 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                           filename       AOA  \\\n",
       "0        1  /home/shubham/PersonalityRecognizer/texts/910.txt  1.364566   \n",
       "1        2  /home/shubham/PersonalityRecognizer/texts/1821... -0.640854   \n",
       "2        3  /home/shubham/PersonalityRecognizer/texts/1679... -1.072180   \n",
       "3        4  /home/shubham/PersonalityRecognizer/texts/2461... -0.659012   \n",
       "4        5  /home/shubham/PersonalityRecognizer/texts/748.txt -1.319069   \n",
       "...    ...                                                ...       ...   \n",
       "2462  2463  /home/shubham/PersonalityRecognizer/texts/1717...  0.259374   \n",
       "2463  2464   /home/shubham/PersonalityRecognizer/texts/52.txt -0.772750   \n",
       "2464  2465  /home/shubham/PersonalityRecognizer/texts/1978... -0.294362   \n",
       "2465  2466  /home/shubham/PersonalityRecognizer/texts/363.txt -0.193252   \n",
       "2466  2467  /home/shubham/PersonalityRecognizer/texts/320.txt  0.536508   \n",
       "\n",
       "     BROWN-FREQ      CONC       FAM      IMAG K-F-FREQ K-F-NCATS K-F-NSAMP  \\\n",
       "0             ? -0.977067  0.768194 -0.562073        ?         ?         ?   \n",
       "1             ? -0.189962 -0.257596  0.056930        ?         ?         ?   \n",
       "2             ?  0.071708 -1.856507  1.156928        ?         ?         ?   \n",
       "3             ?  1.315691 -0.945107  1.666795        ?         ?         ?   \n",
       "4             ?  0.795813 -0.273787  1.163599        ?         ?         ?   \n",
       "...         ...       ...       ...       ...      ...       ...       ...   \n",
       "2462          ?  1.088820 -2.174183  1.571568        ?         ?         ?   \n",
       "2463          ?  0.432476  0.334986  0.215733        ?         ?         ?   \n",
       "2464          ?  0.434861  0.047322  0.704492        ?         ?         ?   \n",
       "2465          ?  0.285823 -0.405907  0.815238        ?         ?         ?   \n",
       "2466          ? -0.135137 -0.531425 -0.539247        ?         ?         ?   \n",
       "\n",
       "      ...  Quote  Apostro  Parenth    OtherP    AllPct Extraversion  \\\n",
       "0     ...      0        0        0 -0.193295 -0.610687     4.796426   \n",
       "1     ...      0        0        0 -0.193295  0.283737     4.370836   \n",
       "2     ...      0        0        0 -0.193295 -2.628056     4.678106   \n",
       "3     ...      0        0        0  1.521484 -0.669396     4.945070   \n",
       "4     ...      0        0        0 -0.193295  0.209775     4.370836   \n",
       "...   ...    ...      ...      ...       ...       ...          ...   \n",
       "2462  ...      0        0        0 -0.193295 -0.077060     4.945070   \n",
       "2463  ...      0        0        0 -0.193295  0.068354     4.678106   \n",
       "2464  ...      0        0        0 -0.193295 -1.619049     4.945070   \n",
       "2465  ...      0        0        0 -0.193295 -0.775789     4.646799   \n",
       "2466  ...      0        0        0  0.184312  0.552863     4.710589   \n",
       "\n",
       "      'Emotional  Agreeableness  Conscientiousness  'Openness  \n",
       "0       4.213128       4.492554           4.332998   4.903835  \n",
       "1       4.052020       4.284788           4.893034   4.973424  \n",
       "2       3.585848       4.284788           4.893034   4.973424  \n",
       "3       3.585848       4.492554           4.819516   4.973424  \n",
       "4       3.767593       4.492554           4.472535   4.253278  \n",
       "...          ...            ...                ...        ...  \n",
       "2462    3.585848       4.711398           4.497862   4.973424  \n",
       "2463    4.213128       4.655560           4.893034   4.681388  \n",
       "2464    3.767593       4.655560           4.893034   4.681388  \n",
       "2465    3.584963       4.653380           4.893034   4.583562  \n",
       "2466    3.767593       4.534645           4.792520   4.857327  \n",
       "\n",
       "[2467 rows x 91 columns]"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/shubham/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "for i in range(len(m['id'])):\n",
    "    m['filename'][i] = m['filename'][i][42:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>filename</th>\n",
       "      <th>AOA</th>\n",
       "      <th>BROWN-FREQ</th>\n",
       "      <th>CONC</th>\n",
       "      <th>FAM</th>\n",
       "      <th>IMAG</th>\n",
       "      <th>K-F-FREQ</th>\n",
       "      <th>K-F-NCATS</th>\n",
       "      <th>K-F-NSAMP</th>\n",
       "      <th>...</th>\n",
       "      <th>Quote</th>\n",
       "      <th>Apostro</th>\n",
       "      <th>Parenth</th>\n",
       "      <th>OtherP</th>\n",
       "      <th>AllPct</th>\n",
       "      <th>Extraversion</th>\n",
       "      <th>'Emotional</th>\n",
       "      <th>Agreeableness</th>\n",
       "      <th>Conscientiousness</th>\n",
       "      <th>'Openness</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>910.txt</td>\n",
       "      <td>1.364566</td>\n",
       "      <td>?</td>\n",
       "      <td>-0.977067</td>\n",
       "      <td>0.768194</td>\n",
       "      <td>-0.562073</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.193295</td>\n",
       "      <td>-0.610687</td>\n",
       "      <td>4.796426</td>\n",
       "      <td>4.213128</td>\n",
       "      <td>4.492554</td>\n",
       "      <td>4.332998</td>\n",
       "      <td>4.903835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1821.txt</td>\n",
       "      <td>-0.640854</td>\n",
       "      <td>?</td>\n",
       "      <td>-0.189962</td>\n",
       "      <td>-0.257596</td>\n",
       "      <td>0.056930</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.193295</td>\n",
       "      <td>0.283737</td>\n",
       "      <td>4.370836</td>\n",
       "      <td>4.052020</td>\n",
       "      <td>4.284788</td>\n",
       "      <td>4.893034</td>\n",
       "      <td>4.973424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1679.txt</td>\n",
       "      <td>-1.072180</td>\n",
       "      <td>?</td>\n",
       "      <td>0.071708</td>\n",
       "      <td>-1.856507</td>\n",
       "      <td>1.156928</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.193295</td>\n",
       "      <td>-2.628056</td>\n",
       "      <td>4.678106</td>\n",
       "      <td>3.585848</td>\n",
       "      <td>4.284788</td>\n",
       "      <td>4.893034</td>\n",
       "      <td>4.973424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>2461.txt</td>\n",
       "      <td>-0.659012</td>\n",
       "      <td>?</td>\n",
       "      <td>1.315691</td>\n",
       "      <td>-0.945107</td>\n",
       "      <td>1.666795</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.521484</td>\n",
       "      <td>-0.669396</td>\n",
       "      <td>4.945070</td>\n",
       "      <td>3.585848</td>\n",
       "      <td>4.492554</td>\n",
       "      <td>4.819516</td>\n",
       "      <td>4.973424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>748.txt</td>\n",
       "      <td>-1.319069</td>\n",
       "      <td>?</td>\n",
       "      <td>0.795813</td>\n",
       "      <td>-0.273787</td>\n",
       "      <td>1.163599</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.193295</td>\n",
       "      <td>0.209775</td>\n",
       "      <td>4.370836</td>\n",
       "      <td>3.767593</td>\n",
       "      <td>4.492554</td>\n",
       "      <td>4.472535</td>\n",
       "      <td>4.253278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2462</th>\n",
       "      <td>2463</td>\n",
       "      <td>1717.txt</td>\n",
       "      <td>0.259374</td>\n",
       "      <td>?</td>\n",
       "      <td>1.088820</td>\n",
       "      <td>-2.174183</td>\n",
       "      <td>1.571568</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.193295</td>\n",
       "      <td>-0.077060</td>\n",
       "      <td>4.945070</td>\n",
       "      <td>3.585848</td>\n",
       "      <td>4.711398</td>\n",
       "      <td>4.497862</td>\n",
       "      <td>4.973424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2463</th>\n",
       "      <td>2464</td>\n",
       "      <td>52.txt</td>\n",
       "      <td>-0.772750</td>\n",
       "      <td>?</td>\n",
       "      <td>0.432476</td>\n",
       "      <td>0.334986</td>\n",
       "      <td>0.215733</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.193295</td>\n",
       "      <td>0.068354</td>\n",
       "      <td>4.678106</td>\n",
       "      <td>4.213128</td>\n",
       "      <td>4.655560</td>\n",
       "      <td>4.893034</td>\n",
       "      <td>4.681388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2464</th>\n",
       "      <td>2465</td>\n",
       "      <td>1978.txt</td>\n",
       "      <td>-0.294362</td>\n",
       "      <td>?</td>\n",
       "      <td>0.434861</td>\n",
       "      <td>0.047322</td>\n",
       "      <td>0.704492</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.193295</td>\n",
       "      <td>-1.619049</td>\n",
       "      <td>4.945070</td>\n",
       "      <td>3.767593</td>\n",
       "      <td>4.655560</td>\n",
       "      <td>4.893034</td>\n",
       "      <td>4.681388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2465</th>\n",
       "      <td>2466</td>\n",
       "      <td>363.txt</td>\n",
       "      <td>-0.193252</td>\n",
       "      <td>?</td>\n",
       "      <td>0.285823</td>\n",
       "      <td>-0.405907</td>\n",
       "      <td>0.815238</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.193295</td>\n",
       "      <td>-0.775789</td>\n",
       "      <td>4.646799</td>\n",
       "      <td>3.584963</td>\n",
       "      <td>4.653380</td>\n",
       "      <td>4.893034</td>\n",
       "      <td>4.583562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2466</th>\n",
       "      <td>2467</td>\n",
       "      <td>320.txt</td>\n",
       "      <td>0.536508</td>\n",
       "      <td>?</td>\n",
       "      <td>-0.135137</td>\n",
       "      <td>-0.531425</td>\n",
       "      <td>-0.539247</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.184312</td>\n",
       "      <td>0.552863</td>\n",
       "      <td>4.710589</td>\n",
       "      <td>3.767593</td>\n",
       "      <td>4.534645</td>\n",
       "      <td>4.792520</td>\n",
       "      <td>4.857327</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2467 rows × 91 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        id  filename       AOA BROWN-FREQ      CONC       FAM      IMAG  \\\n",
       "0        1   910.txt  1.364566          ? -0.977067  0.768194 -0.562073   \n",
       "1        2  1821.txt -0.640854          ? -0.189962 -0.257596  0.056930   \n",
       "2        3  1679.txt -1.072180          ?  0.071708 -1.856507  1.156928   \n",
       "3        4  2461.txt -0.659012          ?  1.315691 -0.945107  1.666795   \n",
       "4        5   748.txt -1.319069          ?  0.795813 -0.273787  1.163599   \n",
       "...    ...       ...       ...        ...       ...       ...       ...   \n",
       "2462  2463  1717.txt  0.259374          ?  1.088820 -2.174183  1.571568   \n",
       "2463  2464    52.txt -0.772750          ?  0.432476  0.334986  0.215733   \n",
       "2464  2465  1978.txt -0.294362          ?  0.434861  0.047322  0.704492   \n",
       "2465  2466   363.txt -0.193252          ?  0.285823 -0.405907  0.815238   \n",
       "2466  2467   320.txt  0.536508          ? -0.135137 -0.531425 -0.539247   \n",
       "\n",
       "     K-F-FREQ K-F-NCATS K-F-NSAMP  ...  Quote  Apostro  Parenth    OtherP  \\\n",
       "0           ?         ?         ?  ...      0        0        0 -0.193295   \n",
       "1           ?         ?         ?  ...      0        0        0 -0.193295   \n",
       "2           ?         ?         ?  ...      0        0        0 -0.193295   \n",
       "3           ?         ?         ?  ...      0        0        0  1.521484   \n",
       "4           ?         ?         ?  ...      0        0        0 -0.193295   \n",
       "...       ...       ...       ...  ...    ...      ...      ...       ...   \n",
       "2462        ?         ?         ?  ...      0        0        0 -0.193295   \n",
       "2463        ?         ?         ?  ...      0        0        0 -0.193295   \n",
       "2464        ?         ?         ?  ...      0        0        0 -0.193295   \n",
       "2465        ?         ?         ?  ...      0        0        0 -0.193295   \n",
       "2466        ?         ?         ?  ...      0        0        0  0.184312   \n",
       "\n",
       "        AllPct Extraversion  'Emotional  Agreeableness  Conscientiousness  \\\n",
       "0    -0.610687     4.796426    4.213128       4.492554           4.332998   \n",
       "1     0.283737     4.370836    4.052020       4.284788           4.893034   \n",
       "2    -2.628056     4.678106    3.585848       4.284788           4.893034   \n",
       "3    -0.669396     4.945070    3.585848       4.492554           4.819516   \n",
       "4     0.209775     4.370836    3.767593       4.492554           4.472535   \n",
       "...        ...          ...         ...            ...                ...   \n",
       "2462 -0.077060     4.945070    3.585848       4.711398           4.497862   \n",
       "2463  0.068354     4.678106    4.213128       4.655560           4.893034   \n",
       "2464 -1.619049     4.945070    3.767593       4.655560           4.893034   \n",
       "2465 -0.775789     4.646799    3.584963       4.653380           4.893034   \n",
       "2466  0.552863     4.710589    3.767593       4.534645           4.792520   \n",
       "\n",
       "      'Openness  \n",
       "0      4.903835  \n",
       "1      4.973424  \n",
       "2      4.973424  \n",
       "3      4.973424  \n",
       "4      4.253278  \n",
       "...         ...  \n",
       "2462   4.973424  \n",
       "2463   4.681388  \n",
       "2464   4.681388  \n",
       "2465   4.583562  \n",
       "2466   4.857327  \n",
       "\n",
       "[2467 rows x 91 columns]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "m.to_csv('mairesse_feat_m5reg.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}