QA Final.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import the necessary packages."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "import json\n",
    "import unicodecsv as csv\n",
    "import numpy as np\n",
    "import math\n",
    "import re\n",
    "import operator\n",
    "from collections import defaultdict\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from scipy.spatial.distance import cosine as cos_distance\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.tokenize import sent_tokenize\n",
    "from nltk.stem import wordnet\n",
    "from nltk import pos_tag\n",
    "import string\n",
    "from sklearn.metrics import f1_score\n",
    "import tensorflow as tf\n",
    "from nltk.tag.stanford import StanfordNERTagger\n",
    "from sklearn.model_selection import train_test_split\n",
    "import spacy\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.tree import DecisionTreeClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Init the lists for storing trianing data, test data, and documents."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_set = []  # list of dict with questions, answers, answer_para, docid\n",
    "test_set = []  # list of sorted dict with questions, docid, id (question id, not using)\n",
    "devel_set = []  # list of dict with questions, answers, answer_para, docid\n",
    "docs = []  # list of sorted dict with docid and text (list of paras)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read all the data from the .json files and store in the corresponding lists。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read documents data\n",
    "with open('documents.json') as doc_data:\n",
    "    docs = json.load(doc_data)  # length = 441\n",
    "doc_data.close()\n",
    "\n",
    "# Read training data\n",
    "with open('training.json') as training_data:\n",
    "    training_set = json.load(training_data)  # length = 43379\n",
    "training_data.close()\n",
    "\n",
    "# Read test data\n",
    "with open('testing.json') as test_data:\n",
    "    test_set = json.load(test_data)  # length = 3618\n",
    "test_data.close()\n",
    "\n",
    "# Read develop data\n",
    "with open('devel.json') as devel_data:\n",
    "    devel_set = json.load(devel_data)  # length = 3618\n",
    "devel_data.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section 2: Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define a function to tokenize, convert to lowercase, remove the stop words and punctuations, lemmatize the words, and finally get the BOW for the given text (a string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_words = set(stopwords.words('english'))\n",
    "lemmatizer = wordnet.WordNetLemmatizer()\n",
    "\n",
    "def get_BOW(text):\n",
    "    word_ls = word_tokenize(text)  # tokenize\n",
    "    new_ls = []\n",
    "    for word in word_ls:\n",
    "        word = word.lower()  # to lower case\n",
    "        if ((word not in stop_words) and not (re.match(\"[^\\w\\s]\", word))):  # remove stop words and punctuations\n",
    "            lemma1 = lemmatizer.lemmatize(word,\"v\")\n",
    "            lemma2 = lemmatizer.lemmatize(word,\"n\")\n",
    "            if (len(lemma1) < len(lemma2)):\n",
    "                new_ls.append(lemma1)\n",
    "            else:\n",
    "                new_ls.append(lemma2)\n",
    "        \n",
    "    # get BOW\n",
    "    BOW = {}\n",
    "    for word in new_ls:\n",
    "        BOW[word] = BOW.get(word,0) + 1\n",
    "    return BOW"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define a function to compute the inverted index the document. The inverted index find the (para_id, weight) pairs for each term, where weight = normalised tf-idf."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_inverted_index(text):\n",
    "    # tf\n",
    "    tf_ls = []\n",
    "    for para in text: # get BOW for each paragraph\n",
    "        para_tf = get_BOW(para)\n",
    "        tf_ls.append(para_tf)\n",
    "    \n",
    "    # df\n",
    "    df = {}  # paragraph frequency of the terms\n",
    "    for para_tf in tf_ls:\n",
    "        for term in para_tf.keys():\n",
    "            df[term] = df.get(term,0) + 1\n",
    "            \n",
    "    M = len(tf_ls)  # total number of paragraphs in the doc\n",
    "    \n",
    "    # inverted index\n",
    "    inverted_index = defaultdict(list)  # term -> (docid,weight) pairs\n",
    "    \n",
    "    # tf-idf values\n",
    "    for para_id in range(len(tf_ls)):\n",
    "        para_tf = tf_ls[para_id]\n",
    "        N = sum(para_tf.values())  # row sum\n",
    "        vec_len = 0  # vector length\n",
    "        \n",
    "        tfidf = {}\n",
    "        for (term,count) in para_tf.items():\n",
    "            tfidf_value = float(count) / N * math.log(M / float(df[term]))\n",
    "            tfidf[term] = tfidf_value\n",
    "            vec_len += pow(tfidf_value,2)  # squared sum for euclidean distance\n",
    "        \n",
    "        # normalise paragraph by vector length and insert into index\n",
    "        vec_len = pow(vec_len,0.5)   # sqrt for euclidean distance\n",
    "        for (term, tfidf_value) in tfidf.items():\n",
    "            if vec_len == 0:\n",
    "                weight = 0\n",
    "            else:\n",
    "                weight = tfidf_value/vec_len\n",
    "            inverted_index[term].append([para_id,weight])  # weight = normalised tf-idf\n",
    "    return inverted_index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get the inverted index for each document, and store as a new attribute."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "for doc in docs:\n",
    "    doc[\"inverted_index\"] = get_inverted_index(doc[\"text\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " Define a function to solve for the para_id with highest score in the document for each the question (query).\n",
    " If no paragraph contains any key words of the query, choose the first paragraph as the best."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def query_vsm(query, inverted_index):\n",
    "    accumulator = {}\n",
    "    for term in query:\n",
    "        postings = inverted_index[term]\n",
    "        for (para_id, weight) in postings:\n",
    "            accumulator[para_id] = accumulator.get(para_id,0) + weight\n",
    "    if (len(accumulator) == 0):  # no paragraph contains query terms, just choose first paragraph\n",
    "        return 0\n",
    "    return max(accumulator.items(), key=operator.itemgetter(1))[0]  # para with max score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define a function to find the best paragraph for each question, and store as a new attribute for each question in the question set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_best_paras(question_set):\n",
    "    for info in question_set:\n",
    "        # Get query terms\n",
    "        query_terms = get_BOW(info[\"question\"]).keys()\n",
    "        \n",
    "        # Get inverted_index for the document\n",
    "        inverted_index = docs[info[\"docid\"]][\"inverted_index\"]\n",
    "        \n",
    "        # Find the index of the best paragraph\n",
    "        info[\"best_para_id\"] = query_vsm(query_terms, inverted_index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define a function to evaluate the prediciton of answer paragraph."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_best_paras(question_set):\n",
    "    correct = 0\n",
    "    for info in question_set:\n",
    "        prediction = info[\"best_para_id\"]\n",
    "        golden_para = info[\"answer_paragraph\"]\n",
    "        if (prediction == golden_para):\n",
    "            correct += 1\n",
    "    return correct*1.0/len(question_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test the paragraph accuracy with the development set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.736842\n"
     ]
    }
   ],
   "source": [
    "# Get best para_id for each question in the development set\n",
    "find_best_paras(devel_set)\n",
    "\n",
    "# Evaluate the prediction\n",
    "accuracy = evaluate_best_paras(devel_set)\n",
    "\n",
    "print(\"accuracy: %f\" % (accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section 3: Common Methods for Basic and Advanced methods"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Write a function to evaluate with average F-score. (Shared by both basic and advanced method)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_answer(question_set, answers):\n",
    "    total_score = 0\n",
    "    for i in range(len(question_set)):\n",
    "        prediction_tokens = word_tokenize(answers[i])\n",
    "        golden_standard_tokens = word_tokenize(question_set[i][\"text\"])\n",
    "        \n",
    "        # precision\n",
    "        precision = 0\n",
    "        for word in prediction_tokens:\n",
    "            if word in golden_standard_tokens:\n",
    "                precision += 1\n",
    "        if (len(prediction_tokens) == 0):\n",
    "            precision = 0\n",
    "        else:\n",
    "            precision /= len(prediction_tokens)\n",
    "        \n",
    "        # recall\n",
    "        recall = 0\n",
    "        for word in golden_standard_tokens:\n",
    "            if word in prediction_tokens:\n",
    "                recall += 1\n",
    "        recall /= len(golden_standard_tokens)\n",
    "        \n",
    "        if (recall + precision == 0):\n",
    "            fscore = 0\n",
    "        else:\n",
    "            fscore = 2*(precision*recall)/(precision + recall)\n",
    "        \n",
    "        total_score += fscore\n",
    "    return (total_score/len(question_set))\n",
    "\n",
    "def evaluate_answer_recall(question_set, answers):\n",
    "    total_recall = 0\n",
    "    for i in range(len(question_set)):\n",
    "        prediction_tokens = word_tokenize(answers[i])\n",
    "        golden_standard_tokens = word_tokenize(question_set[i][\"text\"])\n",
    "        \n",
    "        # recall\n",
    "        recall = 0\n",
    "        for word in golden_standard_tokens:\n",
    "            if word in prediction_tokens:\n",
    "                recall += 1\n",
    "        recall /= len(golden_standard_tokens)\n",
    "        \n",
    "        total_recall += recall\n",
    "    return (total_recall/len(question_set))\n",
    "\n",
    "def evaluate_answer_precision(question_set, answers):\n",
    "    total_precision = 0\n",
    "    for i in range(len(question_set)):\n",
    "        prediction_tokens = word_tokenize(answers[i])\n",
    "        golden_standard_tokens = word_tokenize(question_set[i][\"text\"])\n",
    "        \n",
    "        # precision\n",
    "        precision = 0\n",
    "        for word in prediction_tokens:\n",
    "            if word in golden_standard_tokens:\n",
    "                precision += 1\n",
    "        if (len(prediction_tokens) == 0):\n",
    "            precision = 0\n",
    "        else:\n",
    "            precision /= len(prediction_tokens)\n",
    "        \n",
    "        total_precision += precision\n",
    "    return (total_precision/len(question_set))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define a function for writing the answers to the \"result.csv\" file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def writeToFile(answers):\n",
    "    result = []\n",
    "    for i in range(len(answers)):\n",
    "        answer_dict = {}\n",
    "        answer_dict[\"id\"] = i\n",
    "        answer_dict[\"answer\"] = answers[i]\n",
    "        result.append(answer_dict)\n",
    "\n",
    "    with open(\"result.csv\",\"wb\") as myFile:\n",
    "        fieldnames = ['id', 'answer']\n",
    "        writer = csv.DictWriter(myFile, fieldnames=fieldnames)\n",
    "        writer.writeheader()\n",
    "        writer.writerows(result)\n",
    "    myFile.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section 4: Basic Method"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use the same method to find the best sentence index and store in the question_set as a new attribute."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_best_sent(question_set):\n",
    "    for info in question_set:\n",
    "        # Get query terms\n",
    "        query_terms = get_BOW(info[\"question\"]).keys()\n",
    "        \n",
    "        para = docs[info[\"docid\"]][\"text\"][info[\"best_para_id\"]]\n",
    "        sents = sent_tokenize(para)\n",
    "        \n",
    "        # Get inverted_index for the paragraph\n",
    "        inverted_index = get_inverted_index(sents)\n",
    "        \n",
    "        # Find the index of the best paragraph\n",
    "        info[\"best_sent_id\"] = query_vsm(query_terms, inverted_index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define a method to obtain the answers by applying this basic method."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def basic_method(question_set):\n",
    "    # calculate best paragraph and sentences for the question_set\n",
    "    find_best_paras(question_set)\n",
    "    find_best_sent(question_set)\n",
    "    \n",
    "    # Obtain the answers\n",
    "    answers = []\n",
    "    for info in question_set:\n",
    "        para = docs[info[\"docid\"]][\"text\"][info[\"best_para_id\"]]\n",
    "        sents = sent_tokenize(para)\n",
    "        best_sent = sents[info[\"best_sent_id\"]]\n",
    "        query_terms = get_BOW(info[\"question\"]).keys()\n",
    "        \n",
    "        # find keywords in the best sentence\n",
    "        answer_set = set()  # without pos tag conditiion\n",
    "        answer_set_pos = set()  # with pos tag conditiion, more stricted than answer_set_pos\n",
    "        tagged_sent = pos_tag(word_tokenize(best_sent))  # pos tagged sentence\n",
    "        \n",
    "        for (token, tag) in tagged_sent:\n",
    "            if ((token not in stop_words) and (not re.match(\"[^\\w\\s]\", token)) # remove stop words and punctuations\n",
    "                and (token not in query_terms)): # answers usually not in query terms\n",
    "                answer_set.add(token)\n",
    "                if (tag in [\"NNP\", \"CD\", \"NN\", \"NNS\", \"NNPS\", \"FW\"]): # leave nouns and numbers and foreign words only (more stricted)\n",
    "                    answer_set_pos.add(token)\n",
    "                \n",
    "        # answer string\n",
    "        answer_str = \"\"\n",
    "        \n",
    "        if (len(answer_set_pos) != 0):  # use strict one if possible\n",
    "            for word in list(answer_set_pos):\n",
    "                answer_str += word + \" \"\n",
    "        else:\n",
    "            for word in list(answer_set):\n",
    "                answer_str += word + \" \"\n",
    "            \n",
    "        answers.append(answer_str)\n",
    "            \n",
    "    return answers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply this basic method to the development and training set set and evaluate for error analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "basic fscore: 0.09136886623952355\n"
     ]
    }
   ],
   "source": [
    "combined_answers = basic_method(devel_set + training_set)\n",
    "print(\"basic fscore: \" + str(evaluate_answer(devel_set + training_set, combined_answers)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate the average recall and recision for the development set answers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "basic average recall: 0.2570719039832153\n",
      "basic average precision: 0.06325076121638475\n"
     ]
    }
   ],
   "source": [
    "print(\"basic average recall: \" + str(evaluate_answer_recall(devel_set + training_set, combined_answers)))\n",
    "print(\"basic average precision: \" + str(evaluate_answer_precision(devel_set + training_set, combined_answers)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply this basic method to the test set for Kaggle submission. (Score: 0.18149)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_answers = basic_method(test_set)\n",
    "writeToFile(test_answers)  # write updated answers to file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section 5: Advanced Method"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Label the question type of each question and subtype if possible. (what, who, where, how, which, when)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def label_type(question_set):\n",
    "    # question keywords\n",
    "    types = [\"what\", \"who\", \"where\", \"whose\", \"whom\", \"how\", \"which\", \"when\"]\n",
    "    for info in question_set:\n",
    "        \n",
    "        tokenized_question = word_tokenize(info[\"question\"])\n",
    "        tokenized_question = [word.lower() for word in tokenized_question]  # all tokens to lower case\n",
    "        info[\"type\"] = \"unknown\"  # init as unknown\n",
    "        \n",
    "        # labelling question type\n",
    "        for token in tokenized_question:\n",
    "            if token in types:\n",
    "                info[\"type\"] = token\n",
    "                break\n",
    "                \n",
    "        # labelling subtype for what, which, and how questions\n",
    "        if(info[\"type\"] == \"what\" or info[\"type\"] == \"which\" or info[\"type\"] == \"how\"):\n",
    "            info[\"subtype\"] = find_subtype(info[\"type\"], tokenized_question)\n",
    "                \n",
    "        if (info[\"type\"] == \"whose\" or info[\"type\"] == \"whom\"):  # Converting \"whose\" and \"whom\" to \"who\"\n",
    "            info[\"type\"] = \"who\"\n",
    "            \n",
    "        elif (info[\"type\"] == \"unknown\" and (\"or\" in tokenized_question)):  # finding \"or\" questions\n",
    "            info[\"type\"] = \"or\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# first noun in query_terms as subtype name\n",
    "def find_subtype(question_type, tokenized_question):\n",
    "     \n",
    "    # find first noun in query_terms\n",
    "    subtype = \"unknown\"\n",
    "    \n",
    "    index = tokenized_question.index(question_type)  # index of \"what\", \"which\", or \"how\"\n",
    "    part_tokens = tokenized_question[index:]  # only take words after the \"what\", \"which\", or \"how\"\n",
    "    \n",
    "    if question_type == \"how\":  # how: (much) or (number)\n",
    "        num_ls = [\"many\", \"tall\", \"old\", \"long\", \"large\"]\n",
    "        for element in num_ls:\n",
    "            if element == \"much\":\n",
    "                subtype = \"much\"   # can be money\n",
    "            if element in part_tokens:\n",
    "                subtype = \"number\"\n",
    "                break\n",
    "    else:  # what and which\n",
    "        for (token, tag) in pos_tag(part_tokens):\n",
    "            if(tag in [\"NNP\", \"NN\", \"NNS\", \"NNPS\"]):  # noun\n",
    "                subtype = lemmatizer.lemmatize(token, \"n\")\n",
    "                break\n",
    "    \n",
    "    return subtype"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Label the whole training_set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_type(training_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Find the entities for the \"what\" and \"which\" questions in the training set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "what_X = []\n",
    "what_Y = []\n",
    "which_X = []\n",
    "which_Y = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "for info in training_set:\n",
    "    if (info[\"type\"] == \"what\" or info[\"type\"] == \"which\"):\n",
    "        ne_tags = nlp(info[\"text\"]).ents\n",
    "        if (ne_tags):  # can label, then add for training\n",
    "            if (info[\"type\"] == \"what\"):  # what\n",
    "                what_X.append(info[\"subtype\"])\n",
    "                what_Y.append(ne_tags[0].label_)\n",
    "            else:  # which\n",
    "                which_X.append(info[\"subtype\"])\n",
    "                which_Y.append(ne_tags[0].label_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert into X and Y numbers for machine learning."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "answer_type_ls = [\"PERSON\",\"NORP\",\"FAC\",\"ORG\",\"GPE\",\"LOC\",\"PRODUCT\",\"EVENT\",\n",
    "                  \"WORK_OF_ART\",\"LAW\",\"LANGUAGE\",\"DATE\",\"TIME\",\"PERCENT\",\"MONEY\",\n",
    "                  \"QUANTITY\",\"CARDINAL\", \"ORDINAL\"] # all possible values of Y    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "635\n",
      "129\n"
     ]
    }
   ],
   "source": [
    "# all possible subsets for what and which questions\n",
    "what_X_vocab = set()\n",
    "which_X_vocab = set()\n",
    "for x in what_X:\n",
    "    what_X_vocab.add(x)\n",
    "for x in which_X:\n",
    "    which_X_vocab.add(x)\n",
    "    \n",
    "# convert to list\n",
    "what_X_vocab = list(what_X_vocab)\n",
    "which_X_vocab = list(which_X_vocab)\n",
    "\n",
    "print(len(what_X_vocab))\n",
    "print(len(which_X_vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new training data with numbers (index) only\n",
    "new_what_X = []\n",
    "new_what_Y = []\n",
    "new_which_X = []\n",
    "new_which_Y = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "for x in what_X:\n",
    "    one_hot = []\n",
    "    for i in range(len(what_X_vocab)):\n",
    "        if (i == what_X_vocab.index(x)):\n",
    "            one_hot.append(1)\n",
    "        else:\n",
    "            one_hot.append(0)\n",
    "    new_what_X.append(one_hot)\n",
    "for y in what_Y:\n",
    "    new_what_Y.append(answer_type_ls.index(y))\n",
    "for x in which_X:\n",
    "    one_hot = []\n",
    "    for i in range(len(which_X_vocab)):\n",
    "        if (i == which_X_vocab.index(x)):\n",
    "            one_hot.append(1)\n",
    "        else:\n",
    "            one_hot.append(0)\n",
    "    new_which_X.append(one_hot)\n",
    "for y in which_Y:\n",
    "    new_which_Y.append(answer_type_ls.index(y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check the length of the training data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4813\n",
      "4813\n",
      "443\n",
      "443\n"
     ]
    }
   ],
   "source": [
    "print(len(new_what_X))\n",
    "print(len(new_what_Y))\n",
    "print(len(new_which_X))\n",
    "print(len(new_which_Y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training linear GaussianNB classifiers for what and which."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "gnb_what = GaussianNB().fit(new_what_X, new_what_Y)\n",
    "gnb_which = GaussianNB().fit(new_which_X, new_which_Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extract X and Y from development data for testing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_type(devel_set)\n",
    "\n",
    "# What\n",
    "X_develop_what = []\n",
    "true_y_develop_what = []\n",
    "\n",
    "# Which\n",
    "X_develop_which = []\n",
    "true_y_develop_which = []\n",
    "\n",
    "for info in devel_set:\n",
    "    \n",
    "    question_type = info[\"type\"]\n",
    "    \n",
    "    if (question_type == \"what\"):\n",
    "        # predict answer type\n",
    "        subtype = info[\"subtype\"]\n",
    "\n",
    "        if subtype in what_X_vocab:\n",
    "            \n",
    "            # NE tagging\n",
    "            ne_tags = nlp(info[\"text\"]).ents\n",
    "            if (ne_tags):  # can label, then add for training\n",
    "\n",
    "                # Getting true Y\n",
    "                true_y_develop_what.append([answer_type_ls.index(ne_tags[0].label_)])\n",
    "\n",
    "                # Getting X\n",
    "                one_hot = []\n",
    "                for i in range(len(what_X_vocab)):\n",
    "                    if (i == what_X_vocab.index(x)):\n",
    "                        one_hot.append(1)\n",
    "                    else:\n",
    "                        one_hot.append(0)\n",
    "                X_develop_what.append(one_hot)\n",
    "    elif (question_type == \"which\"):\n",
    "        # predict answer type\n",
    "        subtype = info[\"subtype\"]\n",
    "\n",
    "        if subtype in which_X_vocab:\n",
    "            # NE tagging\n",
    "            ne_tags = nlp(info[\"text\"]).ents\n",
    "            if (ne_tags):  # can label, then add for training\n",
    "\n",
    "                # Getting true Y\n",
    "                true_y_develop_which.append([answer_type_ls.index(ne_tags[0].label_)])\n",
    "\n",
    "                # Getting X\n",
    "                one_hot = []\n",
    "                for i in range(len(which_X_vocab)):\n",
    "                    if (i == which_X_vocab.index(x)):\n",
    "                        one_hot.append(1)\n",
    "                    else:\n",
    "                        one_hot.append(0)\n",
    "                X_develop_which.append(one_hot)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training DecisionTreeClassifier for what and which."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtc_what = DecisionTreeClassifier().fit(new_what_X, new_what_Y)\n",
    "dtc_which = DecisionTreeClassifier().fit(new_which_X, new_which_Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Decision tree has better performance comparing with NB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NB what score: 0.0168067226891\n",
      "NB which score: 0.611111111111\n",
      "DT what score: 0.456582633053\n",
      "DT which score: 0.611111111111\n"
     ]
    }
   ],
   "source": [
    "print(\"NB what score: \" + str(gnb_what.score(X_develop_what, true_y_develop_what)))\n",
    "print(\"NB which score: \" + str(gnb_which.score(X_develop_which, true_y_develop_which)))\n",
    "print(\"DT what score: \" + str(dtc_what.score(X_develop_what, true_y_develop_what)))\n",
    "print(\"DT which score: \" + str(dtc_which.score(X_develop_which, true_y_develop_which)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a dict to map each subtype in what and which to a NE tag."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "what_subtype_tag_dict = {}\n",
    "for i in range(len(what_X_vocab)):\n",
    "    one_hot = []\n",
    "    for j in range(len(what_X_vocab)):\n",
    "        if (j == i):\n",
    "            one_hot.append(1)\n",
    "        else:\n",
    "            one_hot.append(0)\n",
    "    predict_y = dtc_what.predict([one_hot])[0]\n",
    "    what_subtype_tag_dict[what_X_vocab[i]] = predict_y\n",
    "\n",
    "which_subtype_tag_dict = {}\n",
    "for i in range(len(which_X_vocab)):\n",
    "    one_hot = []\n",
    "    for j in range(len(which_X_vocab)):\n",
    "        if (j == i):\n",
    "            one_hot.append(1)\n",
    "        else:\n",
    "            one_hot.append(0)\n",
    "    predict_y = dtc_which.predict([one_hot])[0]\n",
    "    which_subtype_tag_dict[which_X_vocab[i]] = predict_y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define a function to filter the answer string according to the NE tags."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_answer(answer_str, tags):\n",
    "    ents = nlp(answer_str).ents\n",
    "    if (ents):  # can label\n",
    "        filtered_tokens = []\n",
    "        for ent in ents:\n",
    "            if ent.label_ in tags:\n",
    "                filtered_tokens.append(ent.text)\n",
    "        if (len(filtered_tokens) != 0):  # target NE found\n",
    "            new_str = \"\"\n",
    "            for token in filtered_tokens:\n",
    "                new_str += token + \" \"\n",
    "            return new_str\n",
    "        else:  # use original answer\n",
    "            return answer_str\n",
    "    else:  # cannot label, use original answer\n",
    "        return answer_str"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dictionary that maps the simple question types to expected answer types."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "question_ne_dict = {\"who\": [\"PERSON\"], \"where\": [\"GPE\", \"LOC\", \"FAC\"], \"when\": [\"DATE\", \"TIME\"]}\n",
    "how_ne_dict = {\"much\": [\"QUANTITY\", \"CARDINAL\", \"MONEY\"], \"number\": [\"QUANTITY\", \"CARDINAL\", \"PERCENT\"]}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the advanced method."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def advanced_method(question_set):\n",
    "    \n",
    "    answers = basic_method(question_set)  # apply basic method first\n",
    "    label_type(question_set)  # label question types\n",
    "    \n",
    "    for i in range(len(question_set)):\n",
    "        info = question_set[i]\n",
    "        question_type = info[\"type\"]\n",
    "        answer = answers[i]  # answers from basic method\n",
    "        \n",
    "        if question_type in [\"who\", \"where\", \"when\"]:  # who, where, when questions\n",
    "            answers[i] = filter_answer(answer, question_ne_dict[question_type])\n",
    "            \n",
    "        elif question_type == \"how\":   # how questions\n",
    "            subtype = info[\"subtype\"]\n",
    "            if subtype in how_ne_dict.keys():\n",
    "                answers[i] = filter_answer(answer, how_ne_dict[subtype])\n",
    "                \n",
    "        elif question_type == \"or\":  # or questions\n",
    "            # Find the best sentence\n",
    "            query_terms = word_tokenize(info[\"question\"])\n",
    "            \n",
    "            # use the words before and after \"or\" as answer\n",
    "            index = query_terms.index(\"or\")\n",
    "            if (index != 0 and index != len(query_terms) - 2):\n",
    "                before = query_terms[index -1]\n",
    "                after = query_terms[index + 1]\n",
    "                answers[i] = before + \" \" + after\n",
    "        \n",
    "        elif question_type == \"what\":  # what questions\n",
    "            # predict answer type\n",
    "            subtype = info[\"subtype\"]\n",
    "            if subtype in what_X_vocab:\n",
    "                answer_tag = answer_type_ls[what_subtype_tag_dict[subtype]]\n",
    "                answers[i] = filter_answer(answer, [answer_tag])\n",
    "                \n",
    "        elif question_type == \"which\":  # which questions\n",
    "            # predict answer type   \n",
    "            subtype = info[\"subtype\"]\n",
    "            if subtype in which_X_vocab:\n",
    "                answer_tag = answer_type_ls[which_subtype_tag_dict[subtype]]\n",
    "                answers[i] = filter_answer(answer, [answer_tag])\n",
    "    return answers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply this basic method to the development set and evaluate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "basic fscore: 0.08337461679547321\n"
     ]
    }
   ],
   "source": [
    "devel_answers_basic = basic_method(devel_set)\n",
    "print(\"basic fscore: \" + str(evaluate_answer(devel_set, devel_answers_basic)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply the basic method to the development set and evaluate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "advanced fscore: 0.11968652329882037\n"
     ]
    }
   ],
   "source": [
    "devel_answers_advanced = advanced_method(devel_set)\n",
    "print(\"advanced fscore: \" + str(evaluate_answer(devel_set, devel_answers_advanced)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply the advanced method to the development set and evaluate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "basic average recall: 0.2420514476374988\n",
      "basic average precision: 0.05696210319977588\n",
      "advanced average recall: 0.22240878269292874\n",
      "advanced average precision: 0.10196751196946854\n"
     ]
    }
   ],
   "source": [
    "print(\"basic average recall: \" + str(evaluate_answer_recall(devel_set, devel_answers_basic)))\n",
    "print(\"basic average precision: \" + str(evaluate_answer_precision(devel_set, devel_answers_basic)))\n",
    "print(\"advanced average recall: \" + str(evaluate_answer_recall(devel_set, devel_answers_advanced)))\n",
    "print(\"advanced average precision: \" + str(evaluate_answer_precision(devel_set, devel_answers_advanced)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply this advanced method to the test set for Kaggle submission. (Score: 0.24050)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_answers = advanced_method(test_set)\n",
    "writeToFile(test_answers)  # write updated answers to file"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}