Virgo_cluster_w2v_d2v_09Dec.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "liiS-J479Gvt"
   },
   "source": [
    "# Alternus Vera - Identify Fake News"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ebnN4ikZzQtu"
   },
   "source": [
    "## Datasets:\n",
    "Original Kaggle fake news dataset: \n",
    "'https://github.com/synle/machine-learning-sample-dataset/raw/master/liar_dataset/kaggle/kaggle-fake.csv'\n",
    "\n",
    "#### This dataset is heavily skewed to fake news. I moved forward to try to find other dataset that enriches non-fake news.\n",
    "\n",
    "Enriched Kaggle news dataset (50,000 verified non-fake news):\n",
    "https://dock2.hyunwookshin.com/public/cmpe257_a1/articles1.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 102
    },
    "colab_type": "code",
    "id": "vymCp8X8IHEk",
    "outputId": "71e6be37-59c8-4f11-c086-eb0564d60a80"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/envs/nlp_venv/lib/python3.6/site-packages/nltk/twitter/__init__.py:20: UserWarning: The twython library has not been installed. Some functionality from the twitter package will not be available.\n",
      "  warnings.warn(\"The twython library has not been installed. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package vader_lexicon to\n",
      "[nltk_data]     /Users/yuxu/nltk_data...\n",
      "[nltk_data]   Package vader_lexicon is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to /Users/yuxu/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to /Users/yuxu/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /Users/yuxu/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dependencies\n",
    "import pandas as pd\n",
    "import nltk\n",
    "import numpy as np\n",
    "import io\n",
    "import requests\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from nltk import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "# from sklearn.pipeline import Pipeline\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.feature_selection import chi2\n",
    "from string import punctuation\n",
    "from nltk import PorterStemmer\n",
    "import copy \n",
    "import re, math\n",
    "from sklearn.model_selection import train_test_split\n",
    "from nltk import WordNetLemmatizer\n",
    "from nltk import bigrams\n",
    "from nltk.util import ngrams\n",
    "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
    "from gensim.models.ldamodel import LdaModel\n",
    "from gensim.models import Word2Vec, Doc2Vec\n",
    "from gensim.corpora import Dictionary\n",
    "import pickle\n",
    "from collections import Counter\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, roc_auc_score\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "nltk.download('vader_lexicon')\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zmVds2371DXz"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/envs/nlp_venv/lib/python3.6/site-packages/urllib3/connectionpool.py:857: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n",
      "/anaconda3/envs/nlp_venv/lib/python3.6/site-packages/urllib3/connectionpool.py:857: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    }
   ],
   "source": [
    "def get_parsed_data2(url):\n",
    "    return pd.read_csv(io.StringIO(requests.get(url, verify=False).content.decode('utf-8')), sep=',', header='infer')\n",
    "\n",
    "# download and parse the dataset...\n",
    "data_kg_fake_news = get_parsed_data2('https://github.com/synle/machine-learning-sample-dataset/raw/master/liar_dataset/kaggle/kaggle-fake.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VmVjSdSC5-DC",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/envs/nlp_venv/lib/python3.6/site-packages/urllib3/connectionpool.py:857: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    }
   ],
   "source": [
    "data_kg_nonfake_news = get_parsed_data2('https://dock2.hyunwookshin.com/public/cmpe257_a1/articles1.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sq5Yu8r9CuLP"
   },
   "outputs": [],
   "source": [
    "def tokenize2(text):\n",
    "    cachedStopWords = set(stopwords.words('english') + list(punctuation))\n",
    "    min_length = 3\n",
    "    # tokenize\n",
    "    # convert to lower case\n",
    "    words = map(lambda word: word.lower(), word_tokenize(text))\n",
    "    # remove stop words\n",
    "    words = [word for word in words if word not in cachedStopWords]\n",
    "    # steming\n",
    "    tokens = list(map(lambda token: PorterStemmer().stem(token), words))\n",
    "    # lemmatize\n",
    "    lemmas = [WordNetLemmatizer().lemmatize(word) for word in tokens]\n",
    "    # only focus on alphabetic words\n",
    "    p = re.compile('[a-zA-Z]+')\n",
    "    \n",
    "    filtered_lemmas = list(filter(lambda lemma: p.match(lemma) and len(lemma) >= min_length, lemmas))\n",
    "    return filtered_lemmas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aQT_pvlW5-DF"
   },
   "outputs": [],
   "source": [
    "data_kg_nonfake_news.rename(columns={\"content\": \"text\"}, inplace=True)\n",
    "data_kg_nonfake_news['type'] = 0\n",
    "data_kg_fake_news.loc[data_kg_fake_news['type']!='bs', 'type'] = 0\n",
    "data_kg_fake_news.loc[data_kg_fake_news['type']=='bs', 'type'] = 1\n",
    "all_data = pd.concat([data_kg_fake_news[['title','text','type']], data_kg_nonfake_news[['title','text','type']]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iqgboZrA5-Da"
   },
   "outputs": [],
   "source": [
    "all_data['text_clean']=all_data['text'].astype('U').apply(tokenize2)\n",
    "all_data['title_clean']=all_data['title'].astype('U').apply(tokenize2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    51507\n",
       "1    11492\n",
       "Name: type, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data.type.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Word2Vec(all_data.text_clean, size=50)\n",
    "w2v_trained = dict(zip(model.wv.index2word, model.wv.vectors))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EmbeddingVectorizer(object):\n",
    "    def __init__(self, word2vec):\n",
    "        self.word2vec = word2vec\n",
    "\n",
    "    def transform(self, X):\n",
    "        return X.apply(self.line_to_vec)\n",
    "    \n",
    "    def line_to_vec(self, line):\n",
    "        row = []\n",
    "        for w in line:\n",
    "            if w not in self.word2vec:\n",
    "                row+=[0]\n",
    "            else:\n",
    "                row+=[np.mean(self.word2vec[w])]\n",
    "        return row  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/envs/nlp_venv/lib/python3.6/site-packages/numpy/core/fromnumeric.py:2920: RuntimeWarning: Mean of empty slice.\n",
      "  out=out, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "embedding = EmbeddingVectorizer(w2v_trained)\n",
    "\n",
    "all_data['text_w2v_mean'] = embedding.transform(all_data['text_clean']).apply(np.mean)\n",
    "all_data['title_w2v_mean'] = embedding.transform(all_data['title_clean']).apply(np.mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 62999 entries, 0 to 49999\n",
      "Data columns (total 7 columns):\n",
      "title             62319 non-null object\n",
      "text              62953 non-null object\n",
      "type              62999 non-null int64\n",
      "text_clean        62999 non-null object\n",
      "title_clean       62999 non-null object\n",
      "text_w2v_mean     62755 non-null float64\n",
      "title_w2v_mean    62784 non-null float64\n",
      "dtypes: float64(2), int64(1), object(4)\n",
      "memory usage: 3.8+ MB\n"
     ]
    }
   ],
   "source": [
    "all_data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Doc2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "tqdm.pandas(desc=\"progress-bar\")\n",
    "from gensim.models import Doc2Vec\n",
    "from sklearn import utils\n",
    "from gensim.models.doc2vec import TaggedDocument\n",
    "\n",
    "def label_sentences(corpus, label_type):\n",
    "    \"\"\"\n",
    "    Gensim's Doc2Vec implementation requires each document/paragraph to have a label associated with it.\n",
    "    We do this by using the TaggedDocument method. The format will be \"TRAIN_i\" or \"TEST_i\" where \"i\" is\n",
    "    a dummy index of the post.\n",
    "    \"\"\"\n",
    "    labeled = []\n",
    "    for i, v in enumerate(corpus):\n",
    "        label = label_type + '_' + str(i)\n",
    "        labeled.append(TaggedDocument(v, [label]))\n",
    "    return labeled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc2vec_text = label_sentences(all_data.text_clean, 'Text')\n",
    "doc2vec_title = label_sentences(all_data.title_clean, 'Title')\n",
    "doc2vec_all_data = doc2vec_text + doc2vec_title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125998/125998 [00:00<00:00, 2947181.86it/s]\n"
     ]
    }
   ],
   "source": [
    "model_dbow = Doc2Vec(dm=0, vector_size=50, negative=5, min_count=1, alpha=0.065, min_alpha=0.065)\n",
    "model_dbow.build_vocab([x for x in tqdm(doc2vec_all_data)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125998/125998 [00:00<00:00, 2715302.60it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2733151.19it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2722576.69it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2556533.95it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2867232.27it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2945457.11it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2892925.89it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2933798.45it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2938039.15it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2908016.92it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2752554.33it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2953391.20it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2944915.47it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2915621.64it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2968004.15it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2906657.40it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2948135.44it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2993253.75it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2861271.12it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2764013.83it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2767284.81it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2913258.96it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2948217.68it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2883376.61it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2721160.79it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2977200.42it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2923670.54it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2949402.36it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2942161.08it/s]\n",
      "100%|██████████| 125998/125998 [00:00<00:00, 2891058.42it/s]\n"
     ]
    }
   ],
   "source": [
    "from sklearn import utils\n",
    "\n",
    "for epoch in range(30):\n",
    "    model_dbow.train(utils.shuffle([x for x in tqdm(doc2vec_all_data)]), total_examples=len(doc2vec_all_data), epochs=1)\n",
    "    model_dbow.alpha -= 0.002\n",
    "    model_dbow.min_alpha = model_dbow.alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_vectors(model, corpus_size, vectors_size, vectors_type):\n",
    "    \"\"\"\n",
    "    Get vectors from trained doc2vec model\n",
    "    :param doc2vec_model: Trained Doc2Vec model\n",
    "    :param corpus_size: Size of the data\n",
    "    :param vectors_size: Size of the embedding vectors\n",
    "    :param vectors_type: Training or Testing vectors\n",
    "    :return: list of vectors\n",
    "    \"\"\"\n",
    "    vectors = np.zeros((corpus_size, vectors_size))\n",
    "    for i in range(0, corpus_size):\n",
    "        prefix = vectors_type + '_' + str(i)\n",
    "        vectors[i] = model.docvecs[prefix]\n",
    "    return vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data['text_d2v_mean'] = np.mean(get_vectors(model_dbow, len(all_data.text_clean), 50, 'Text'),axis=1)\n",
    "all_data['title_d2v_mean'] = np.mean(get_vectors(model_dbow, len(all_data.title_clean), 50, 'Title'),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "data = all_data[['text_w2v_mean','title_w2v_mean', 'text_d2v_mean', 'title_d2v_mean']].fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_w2v_mean</th>\n",
       "      <th>title_w2v_mean</th>\n",
       "      <th>text_d2v_mean</th>\n",
       "      <th>title_d2v_mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.052476</td>\n",
       "      <td>0.066654</td>\n",
       "      <td>-0.220385</td>\n",
       "      <td>-0.114133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.126095</td>\n",
       "      <td>-0.309628</td>\n",
       "      <td>-0.048451</td>\n",
       "      <td>0.017952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.095904</td>\n",
       "      <td>-0.209579</td>\n",
       "      <td>-0.118836</td>\n",
       "      <td>-0.079925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.027249</td>\n",
       "      <td>-0.071950</td>\n",
       "      <td>-0.038647</td>\n",
       "      <td>-0.168344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.075030</td>\n",
       "      <td>-0.066515</td>\n",
       "      <td>-0.155317</td>\n",
       "      <td>-0.074950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-0.073837</td>\n",
       "      <td>0.147018</td>\n",
       "      <td>-0.039953</td>\n",
       "      <td>-0.079269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>-0.142487</td>\n",
       "      <td>-0.089351</td>\n",
       "      <td>-0.017041</td>\n",
       "      <td>-0.119939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>-0.146828</td>\n",
       "      <td>-0.079208</td>\n",
       "      <td>-0.100968</td>\n",
       "      <td>-0.051552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>-0.206168</td>\n",
       "      <td>-0.037514</td>\n",
       "      <td>-0.029861</td>\n",
       "      <td>-0.000631</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>-0.079425</td>\n",
       "      <td>0.006140</td>\n",
       "      <td>-0.050879</td>\n",
       "      <td>-0.176942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.007999</td>\n",
       "      <td>-0.001401</td>\n",
       "      <td>-0.031632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>-0.058975</td>\n",
       "      <td>0.022764</td>\n",
       "      <td>-0.089929</td>\n",
       "      <td>-0.077097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>-0.095710</td>\n",
       "      <td>-0.035302</td>\n",
       "      <td>0.020241</td>\n",
       "      <td>-0.153371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>-0.093026</td>\n",
       "      <td>-0.045653</td>\n",
       "      <td>-0.020734</td>\n",
       "      <td>-0.058160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.049404</td>\n",
       "      <td>-0.079222</td>\n",
       "      <td>-0.111268</td>\n",
       "      <td>-0.016399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>-0.051582</td>\n",
       "      <td>0.029093</td>\n",
       "      <td>-0.117301</td>\n",
       "      <td>-0.132241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.049404</td>\n",
       "      <td>-0.038886</td>\n",
       "      <td>-0.109564</td>\n",
       "      <td>-0.058415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>-0.031936</td>\n",
       "      <td>-0.042422</td>\n",
       "      <td>-0.021739</td>\n",
       "      <td>-0.172355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>-0.012848</td>\n",
       "      <td>-0.095763</td>\n",
       "      <td>-0.040788</td>\n",
       "      <td>-0.058262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>-0.037564</td>\n",
       "      <td>0.031969</td>\n",
       "      <td>-0.158895</td>\n",
       "      <td>-0.052953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>-0.021680</td>\n",
       "      <td>-0.020842</td>\n",
       "      <td>-0.055579</td>\n",
       "      <td>-0.014201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>-0.109592</td>\n",
       "      <td>-0.009428</td>\n",
       "      <td>0.020578</td>\n",
       "      <td>-0.047153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.070949</td>\n",
       "      <td>0.087061</td>\n",
       "      <td>-0.044348</td>\n",
       "      <td>-0.104766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>-0.027032</td>\n",
       "      <td>-0.048464</td>\n",
       "      <td>-0.130827</td>\n",
       "      <td>-0.125591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.034479</td>\n",
       "      <td>-0.026224</td>\n",
       "      <td>0.035173</td>\n",
       "      <td>0.015970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>-0.093976</td>\n",
       "      <td>0.001634</td>\n",
       "      <td>-0.161282</td>\n",
       "      <td>-0.045428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>-0.057824</td>\n",
       "      <td>0.024013</td>\n",
       "      <td>0.017568</td>\n",
       "      <td>-0.102108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>-0.017358</td>\n",
       "      <td>0.058224</td>\n",
       "      <td>0.010694</td>\n",
       "      <td>-0.032589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>-0.025687</td>\n",
       "      <td>0.051572</td>\n",
       "      <td>-0.011545</td>\n",
       "      <td>-0.079400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>-0.070981</td>\n",
       "      <td>-0.150526</td>\n",
       "      <td>-0.032012</td>\n",
       "      <td>0.005662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>0.009216</td>\n",
       "      <td>0.005976</td>\n",
       "      <td>-0.155835</td>\n",
       "      <td>-0.111825</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>-0.071957</td>\n",
       "      <td>-0.037876</td>\n",
       "      <td>-0.019945</td>\n",
       "      <td>-0.102049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>-0.040339</td>\n",
       "      <td>0.056044</td>\n",
       "      <td>-0.048194</td>\n",
       "      <td>-0.026885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>-0.059206</td>\n",
       "      <td>0.080694</td>\n",
       "      <td>-0.070887</td>\n",
       "      <td>-0.152047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>-0.087104</td>\n",
       "      <td>-0.082501</td>\n",
       "      <td>-0.071136</td>\n",
       "      <td>-0.118316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>-0.021026</td>\n",
       "      <td>0.063340</td>\n",
       "      <td>0.115047</td>\n",
       "      <td>0.064699</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>-0.058815</td>\n",
       "      <td>-0.069702</td>\n",
       "      <td>-0.091403</td>\n",
       "      <td>-0.092141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>-0.077523</td>\n",
       "      <td>-0.039573</td>\n",
       "      <td>0.038393</td>\n",
       "      <td>-0.066823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>-0.094858</td>\n",
       "      <td>-0.100416</td>\n",
       "      <td>-0.083286</td>\n",
       "      <td>-0.057946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>0.035955</td>\n",
       "      <td>0.061144</td>\n",
       "      <td>-0.056437</td>\n",
       "      <td>-0.098236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>-0.028127</td>\n",
       "      <td>-0.095940</td>\n",
       "      <td>-0.059414</td>\n",
       "      <td>-0.121467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>-0.058970</td>\n",
       "      <td>0.031787</td>\n",
       "      <td>-0.033853</td>\n",
       "      <td>-0.181604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>-0.085513</td>\n",
       "      <td>-0.160682</td>\n",
       "      <td>-0.078869</td>\n",
       "      <td>-0.024612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>-0.071084</td>\n",
       "      <td>-0.057796</td>\n",
       "      <td>-0.078703</td>\n",
       "      <td>-0.048799</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>-0.027994</td>\n",
       "      <td>0.011846</td>\n",
       "      <td>-0.130773</td>\n",
       "      <td>-0.049363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>-0.060544</td>\n",
       "      <td>0.011846</td>\n",
       "      <td>-0.044145</td>\n",
       "      <td>-0.060925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>-0.001237</td>\n",
       "      <td>0.011846</td>\n",
       "      <td>-0.037356</td>\n",
       "      <td>-0.057908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>-0.039331</td>\n",
       "      <td>0.011846</td>\n",
       "      <td>-0.151026</td>\n",
       "      <td>-0.050121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>-0.018864</td>\n",
       "      <td>0.011846</td>\n",
       "      <td>-0.056047</td>\n",
       "      <td>-0.051826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>-0.018864</td>\n",
       "      <td>0.011846</td>\n",
       "      <td>-0.030937</td>\n",
       "      <td>-0.041561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>-0.001237</td>\n",
       "      <td>0.011846</td>\n",
       "      <td>-0.065958</td>\n",
       "      <td>-0.061991</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>-0.039331</td>\n",
       "      <td>0.011846</td>\n",
       "      <td>-0.159414</td>\n",
       "      <td>-0.052447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>-0.018864</td>\n",
       "      <td>0.011846</td>\n",
       "      <td>-0.037972</td>\n",
       "      <td>-0.063573</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>-0.069798</td>\n",
       "      <td>-0.048501</td>\n",
       "      <td>-0.089134</td>\n",
       "      <td>-0.089724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>-0.012393</td>\n",
       "      <td>0.011846</td>\n",
       "      <td>-0.056456</td>\n",
       "      <td>-0.043551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>-0.047423</td>\n",
       "      <td>0.011846</td>\n",
       "      <td>-0.050037</td>\n",
       "      <td>-0.053651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>-0.101742</td>\n",
       "      <td>0.011846</td>\n",
       "      <td>-0.135707</td>\n",
       "      <td>-0.056278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>-0.066955</td>\n",
       "      <td>0.011846</td>\n",
       "      <td>-0.148375</td>\n",
       "      <td>-0.044075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>-0.012393</td>\n",
       "      <td>0.011846</td>\n",
       "      <td>-0.058929</td>\n",
       "      <td>-0.047864</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>-0.047423</td>\n",
       "      <td>0.011846</td>\n",
       "      <td>-0.047653</td>\n",
       "      <td>-0.053796</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>76 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    text_w2v_mean  title_w2v_mean  text_d2v_mean  title_d2v_mean\n",
       "0       -0.052476        0.066654      -0.220385       -0.114133\n",
       "1       -0.126095       -0.309628      -0.048451        0.017952\n",
       "2       -0.095904       -0.209579      -0.118836       -0.079925\n",
       "3       -0.027249       -0.071950      -0.038647       -0.168344\n",
       "4       -0.075030       -0.066515      -0.155317       -0.074950\n",
       "5       -0.073837        0.147018      -0.039953       -0.079269\n",
       "6       -0.142487       -0.089351      -0.017041       -0.119939\n",
       "7       -0.146828       -0.079208      -0.100968       -0.051552\n",
       "8       -0.206168       -0.037514      -0.029861       -0.000631\n",
       "9       -0.079425        0.006140      -0.050879       -0.176942\n",
       "10       0.000000        0.007999      -0.001401       -0.031632\n",
       "11      -0.058975        0.022764      -0.089929       -0.077097\n",
       "12      -0.095710       -0.035302       0.020241       -0.153371\n",
       "13      -0.093026       -0.045653      -0.020734       -0.058160\n",
       "14       0.049404       -0.079222      -0.111268       -0.016399\n",
       "15      -0.051582        0.029093      -0.117301       -0.132241\n",
       "16       0.049404       -0.038886      -0.109564       -0.058415\n",
       "17      -0.031936       -0.042422      -0.021739       -0.172355\n",
       "18      -0.012848       -0.095763      -0.040788       -0.058262\n",
       "19      -0.037564        0.031969      -0.158895       -0.052953\n",
       "20      -0.021680       -0.020842      -0.055579       -0.014201\n",
       "21      -0.109592       -0.009428       0.020578       -0.047153\n",
       "22       0.070949        0.087061      -0.044348       -0.104766\n",
       "23      -0.027032       -0.048464      -0.130827       -0.125591\n",
       "24       0.034479       -0.026224       0.035173        0.015970\n",
       "25      -0.093976        0.001634      -0.161282       -0.045428\n",
       "26      -0.057824        0.024013       0.017568       -0.102108\n",
       "27      -0.017358        0.058224       0.010694       -0.032589\n",
       "28      -0.025687        0.051572      -0.011545       -0.079400\n",
       "29      -0.070981       -0.150526      -0.032012        0.005662\n",
       "..            ...             ...            ...             ...\n",
       "46       0.009216        0.005976      -0.155835       -0.111825\n",
       "47      -0.071957       -0.037876      -0.019945       -0.102049\n",
       "48      -0.040339        0.056044      -0.048194       -0.026885\n",
       "49      -0.059206        0.080694      -0.070887       -0.152047\n",
       "50      -0.087104       -0.082501      -0.071136       -0.118316\n",
       "51      -0.021026        0.063340       0.115047        0.064699\n",
       "52      -0.058815       -0.069702      -0.091403       -0.092141\n",
       "53      -0.077523       -0.039573       0.038393       -0.066823\n",
       "54      -0.094858       -0.100416      -0.083286       -0.057946\n",
       "55       0.035955        0.061144      -0.056437       -0.098236\n",
       "56      -0.028127       -0.095940      -0.059414       -0.121467\n",
       "57      -0.058970        0.031787      -0.033853       -0.181604\n",
       "58      -0.085513       -0.160682      -0.078869       -0.024612\n",
       "59      -0.071084       -0.057796      -0.078703       -0.048799\n",
       "60      -0.027994        0.011846      -0.130773       -0.049363\n",
       "61      -0.060544        0.011846      -0.044145       -0.060925\n",
       "62      -0.001237        0.011846      -0.037356       -0.057908\n",
       "63      -0.039331        0.011846      -0.151026       -0.050121\n",
       "64      -0.018864        0.011846      -0.056047       -0.051826\n",
       "65      -0.018864        0.011846      -0.030937       -0.041561\n",
       "66      -0.001237        0.011846      -0.065958       -0.061991\n",
       "67      -0.039331        0.011846      -0.159414       -0.052447\n",
       "68      -0.018864        0.011846      -0.037972       -0.063573\n",
       "69      -0.069798       -0.048501      -0.089134       -0.089724\n",
       "70      -0.012393        0.011846      -0.056456       -0.043551\n",
       "71      -0.047423        0.011846      -0.050037       -0.053651\n",
       "72      -0.101742        0.011846      -0.135707       -0.056278\n",
       "73      -0.066955        0.011846      -0.148375       -0.044075\n",
       "74      -0.012393        0.011846      -0.058929       -0.047864\n",
       "75      -0.047423        0.011846      -0.047653       -0.053796\n",
       "\n",
       "[76 rows x 4 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(76)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# all_data.to_csv('fake_news_w2v_d2v.csv')\n",
    "data.to_csv('fake_news_w2v_d2v_only.csv',sep=',',header=False)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "YuXu_word2vec_doc2vec_fake_news.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}