MachineLearning Projects/YouTube-Comments-Sentiment-Analysis/YouTube_Comment_Sentiment_Analysis.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "id": "84d0c392",
      "metadata": {
        "id": "84d0c392"
      },
      "source": [
        "# YouTube Comments Sentiment Analysis "
      ]
    },
    {
      "cell_type": "markdown",
      "id": "3bbdd7a6",
      "metadata": {
        "id": "3bbdd7a6"
      },
      "source": [
        "### Import all the necessary packages"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "22a613cd",
      "metadata": {
        "id": "22a613cd"
      },
      "outputs": [],
      "source": [
        "# Basics\n",
        "import pandas as pd; import os\n",
        "import csv; import numpy as np\n",
        "import re; import warnings\n",
        "warnings.filterwarnings('ignore')"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "8a596346",
      "metadata": {
        "id": "8a596346"
      },
      "source": [
        "### Reading Pre-Labeled YouTube Video Comments\n",
        "\n",
        "here we are taking pre-labeled comments of 5 popular youtube videos for training & testing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "fd5c0bcf",
      "metadata": {
        "id": "fd5c0bcf"
      },
      "outputs": [],
      "source": [
        "# training data\n",
        "okgo = pd.read_csv('OKGO.csv', delimiter=\";\", skiprows=2, encoding='latin-1', engine='python') # read in the data\n",
        "trump = pd.read_csv('trump.csv', delimiter=\",\", skiprows=2, encoding='utf-8', error_bad_lines=False, engine='python')\n",
        "swift = pd.read_csv('TaylorSwift.csv', delimiter=\",\", skiprows=2, nrows=180, encoding='utf-8', engine='python')\n",
        "royal = pd.read_csv('RoyalWedding.csv', delimiter=\",\", skiprows=2, nrows=61, encoding='utf-8', engine='python')\n",
        "paul = pd.read_csv('LoganPaul.csv', delimiter=\",\", skiprows=2, nrows=200, encoding='utf-8', engine='python')"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "1928ae40",
      "metadata": {
        "id": "1928ae40"
      },
      "source": [
        "### Reading Pre-Labeled Tweets & Blog Comments"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "264c6049",
      "metadata": {
        "id": "264c6049"
      },
      "outputs": [],
      "source": [
        "blogs = pd.read_csv('Kagel.csv', delimiter=\",\", skiprows=2, encoding='latin-1', engine='python') # read in the data\n",
        "tweets = pd.read_csv('twitter.csv', delimiter=\",\", skiprows=2, encoding='latin-1', engine='python') # read in the data"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "2138f3ca",
      "metadata": {
        "id": "2138f3ca"
      },
      "source": [
        "## Data Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "7eea8c0d",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "7eea8c0d",
        "outputId": "d982863b-454f-49ac-beff-5e1fc85f1657"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  Sentiment                                          TweetText\n",
              "0  positive  Now all @Apple has to do is get swype on the i...\n",
              "1  positive  @Apple will be adding more carrier support to ...\n",
              "2  positive  Hilarious @youtube video - guy does a duet wit...\n",
              "3  positive  @RIM you made it too easy for me to switch to ...\n",
              "4  positive  I just realized that the reason I got into twi..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2e11460b-9b8f-45da-acb0-a6f33a6dc31b\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sentiment</th>\n",
              "      <th>TweetText</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>positive</td>\n",
              "      <td>Now all @Apple has to do is get swype on the i...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>positive</td>\n",
              "      <td>@Apple will be adding more carrier support to ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>positive</td>\n",
              "      <td>Hilarious @youtube video - guy does a duet wit...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>positive</td>\n",
              "      <td>@RIM you made it too easy for me to switch to ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>positive</td>\n",
              "      <td>I just realized that the reason I got into twi...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2e11460b-9b8f-45da-acb0-a6f33a6dc31b')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-2e11460b-9b8f-45da-acb0-a6f33a6dc31b button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-2e11460b-9b8f-45da-acb0-a6f33a6dc31b');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ],
      "source": [
        "# clean dataframes\n",
        "tweets = tweets.drop(['Topic', 'TweetId', \"TweetDate\"], axis = 1).dropna()\n",
        "tweets.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "1a01ffc5",
      "metadata": {
        "id": "1a01ffc5"
      },
      "outputs": [],
      "source": [
        "def fix_cols(DF):\n",
        "    DF = DF.iloc[:,:2]\n",
        "    DF.columns = [\"label\", \"comment\"]\n",
        "    return DF"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d8820145",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "d8820145",
        "outputId": "8fe285f4-4455-40db-f2c7-6088dae9c152"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   label                                            comment\n",
              "0   -1.0  Everyone knows brand's papers from.\\rBut -No o...\n",
              "1    0.0       ÒYour paper cut balance is: \\r-£25279102771Ó\n",
              "2    1.0  OH SHIT WHEN I SAW THIS ON MY FRONT PAGE.........\n",
              "3    1.0                          Blowing my mind yet again\n",
              "4    0.0               Should have gone with Dunder Mifflin"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-00127868-3095-43ea-b7e4-a0efff35082c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>comment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-1.0</td>\n",
              "      <td>Everyone knows brand's papers from.\\rBut -No o...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.0</td>\n",
              "      <td>ÒYour paper cut balance is: \\r-£25279102771Ó</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.0</td>\n",
              "      <td>OH SHIT WHEN I SAW THIS ON MY FRONT PAGE.........</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.0</td>\n",
              "      <td>Blowing my mind yet again</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.0</td>\n",
              "      <td>Should have gone with Dunder Mifflin</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-00127868-3095-43ea-b7e4-a0efff35082c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-00127868-3095-43ea-b7e4-a0efff35082c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-00127868-3095-43ea-b7e4-a0efff35082c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ],
      "source": [
        "okgo = fix_cols(okgo)\n",
        "trump = fix_cols(trump)\n",
        "swift = fix_cols(swift)\n",
        "royal = fix_cols(royal)\n",
        "paul = fix_cols(paul)\n",
        "tweets = fix_cols(tweets)\n",
        "\n",
        "\n",
        "okgo.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "533447a4",
      "metadata": {
        "id": "533447a4"
      },
      "outputs": [],
      "source": [
        "tweets.label = tweets.label.replace({'positive': '1.0', 'negative':'-1.0', 'neutral': '0.0', 'irrelevant': '0.0'}, regex=True)\n",
        "tweets['label'] = pd.to_numeric(tweets['label'], errors='coerce')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "0e128ef2",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "0e128ef2",
        "outputId": "01fc389c-eb55-4105-c16e-c872df85433c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   label                                            comment\n",
              "0    1.0  Now all @Apple has to do is get swype on the i...\n",
              "1    1.0  @Apple will be adding more carrier support to ...\n",
              "2    1.0  Hilarious @youtube video - guy does a duet wit...\n",
              "3    1.0  @RIM you made it too easy for me to switch to ...\n",
              "4    1.0  I just realized that the reason I got into twi..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-9ba36107-3cde-424d-b133-aa474ae33bf6\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>comment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1.0</td>\n",
              "      <td>Now all @Apple has to do is get swype on the i...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.0</td>\n",
              "      <td>@Apple will be adding more carrier support to ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.0</td>\n",
              "      <td>Hilarious @youtube video - guy does a duet wit...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.0</td>\n",
              "      <td>@RIM you made it too easy for me to switch to ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1.0</td>\n",
              "      <td>I just realized that the reason I got into twi...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-9ba36107-3cde-424d-b133-aa474ae33bf6')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-9ba36107-3cde-424d-b133-aa474ae33bf6 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-9ba36107-3cde-424d-b133-aa474ae33bf6');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ],
      "source": [
        "tweets = fix_cols(tweets)\n",
        "blogs = fix_cols(blogs)\n",
        "\n",
        "tweets.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "5e8b608f",
      "metadata": {
        "id": "5e8b608f"
      },
      "source": [
        "### Create Datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e909aaa4",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "e909aaa4",
        "outputId": "4964fd4a-5ac6-4dec-d864-37c80e03a1d7"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   label                                            comment\n",
              "0   -1.0  Everyone knows brand's papers from.\\rBut -No o...\n",
              "1    0.0       ÒYour paper cut balance is: \\r-£25279102771Ó\n",
              "2    1.0  OH SHIT WHEN I SAW THIS ON MY FRONT PAGE.........\n",
              "3    1.0                          Blowing my mind yet again\n",
              "4    0.0               Should have gone with Dunder Mifflin"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-9f1a5dae-eb95-4cca-b822-c1f9c48de8e1\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>comment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-1.0</td>\n",
              "      <td>Everyone knows brand's papers from.\\rBut -No o...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.0</td>\n",
              "      <td>ÒYour paper cut balance is: \\r-£25279102771Ó</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.0</td>\n",
              "      <td>OH SHIT WHEN I SAW THIS ON MY FRONT PAGE.........</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.0</td>\n",
              "      <td>Blowing my mind yet again</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.0</td>\n",
              "      <td>Should have gone with Dunder Mifflin</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-9f1a5dae-eb95-4cca-b822-c1f9c48de8e1')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-9f1a5dae-eb95-4cca-b822-c1f9c48de8e1 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-9f1a5dae-eb95-4cca-b822-c1f9c48de8e1');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ],
      "source": [
        "yt_comments = pd.concat([okgo, trump, swift, royal, paul], ignore_index=True)\n",
        "yt_comments.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "92cf2f3c",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "92cf2f3c",
        "outputId": "b022cd77-a745-4f52-e753-8fc2d0a25222"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   label                                            comment\n",
              "0    1.0                    i liked the Da Vinci Code a lot\n",
              "1    1.0                    i liked the Da Vinci Code a lot\n",
              "2    1.0   I liked the Da Vinci Code but it ultimatly di...\n",
              "3    1.0   that's not even an exaggeration ) and at midn...\n",
              "4    1.0   I loved the Da Vinci Code but now I want some..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-3875d792-8b16-4c25-b7d6-6e229fb2b3de\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>comment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1.0</td>\n",
              "      <td>i liked the Da Vinci Code a lot</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.0</td>\n",
              "      <td>i liked the Da Vinci Code a lot</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.0</td>\n",
              "      <td>I liked the Da Vinci Code but it ultimatly di...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.0</td>\n",
              "      <td>that's not even an exaggeration ) and at midn...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1.0</td>\n",
              "      <td>I loved the Da Vinci Code but now I want some...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3875d792-8b16-4c25-b7d6-6e229fb2b3de')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-3875d792-8b16-4c25-b7d6-6e229fb2b3de button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-3875d792-8b16-4c25-b7d6-6e229fb2b3de');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ],
      "source": [
        "non_yt_comments = pd.concat([blogs, tweets], ignore_index=True)\n",
        "non_yt_comments.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "cee5a6fe",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "cee5a6fe",
        "outputId": "770d05d4-33bd-430d-9ea7-fcbe62ca33d1"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   label                                            comment\n",
              "0   -1.0  Everyone knows brand's papers from.\\rBut -No o...\n",
              "1    0.0       ÒYour paper cut balance is: \\r-£25279102771Ó\n",
              "2    1.0  OH SHIT WHEN I SAW THIS ON MY FRONT PAGE.........\n",
              "3    1.0                          Blowing my mind yet again\n",
              "4    0.0               Should have gone with Dunder Mifflin"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-46b6f338-ffa7-4439-bee7-2ec9c1b052a0\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>comment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-1.0</td>\n",
              "      <td>Everyone knows brand's papers from.\\rBut -No o...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.0</td>\n",
              "      <td>ÒYour paper cut balance is: \\r-£25279102771Ó</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.0</td>\n",
              "      <td>OH SHIT WHEN I SAW THIS ON MY FRONT PAGE.........</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.0</td>\n",
              "      <td>Blowing my mind yet again</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.0</td>\n",
              "      <td>Should have gone with Dunder Mifflin</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-46b6f338-ffa7-4439-bee7-2ec9c1b052a0')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-46b6f338-ffa7-4439-bee7-2ec9c1b052a0 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-46b6f338-ffa7-4439-bee7-2ec9c1b052a0');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ],
      "source": [
        "comments = pd.concat([yt_comments, non_yt_comments], ignore_index=True)\n",
        "comments.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "b13d25d3",
      "metadata": {
        "id": "b13d25d3"
      },
      "source": [
        "### Remove Non-Alphabetic Characters (including numbers)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "04216102",
      "metadata": {
        "id": "04216102"
      },
      "outputs": [],
      "source": [
        "def convert_to_string(DF):\n",
        "    DF[\"comment\"]= DF[\"comment\"].astype(str) #object to string data type"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "dcc4e3bb",
      "metadata": {
        "id": "dcc4e3bb"
      },
      "outputs": [],
      "source": [
        "convert_to_string(comments)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "fd9cfdcd",
      "metadata": {
        "id": "fd9cfdcd"
      },
      "outputs": [],
      "source": [
        "def cleanerFn(b):\n",
        "    # keeps only words with alphabetic characters in comments\n",
        "    for row in range(len(b)):\n",
        "        line = b.loc[row, \"comment\"]\n",
        "        b.loc[row,\"comment\"] = re.sub(\"[^a-zA-Z]\", \" \", line)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "50219d27",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "50219d27",
        "outputId": "cc2150ec-f9a5-4dd8-b9ed-560d89bf960b"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   label                                            comment\n",
              "0   -1.0  Everyone knows brand s papers from  But  No on...\n",
              "1    0.0         Your paper cut balance is                 \n",
              "2    1.0  OH SHIT WHEN I SAW THIS ON MY FRONT PAGE      ...\n",
              "3    1.0                          Blowing my mind yet again\n",
              "4    0.0               Should have gone with Dunder Mifflin"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-089bbc04-cf1b-4dbe-8ceb-64211b06c865\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>comment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-1.0</td>\n",
              "      <td>Everyone knows brand s papers from  But  No on...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.0</td>\n",
              "      <td>Your paper cut balance is</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.0</td>\n",
              "      <td>OH SHIT WHEN I SAW THIS ON MY FRONT PAGE      ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.0</td>\n",
              "      <td>Blowing my mind yet again</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.0</td>\n",
              "      <td>Should have gone with Dunder Mifflin</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-089bbc04-cf1b-4dbe-8ceb-64211b06c865')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-089bbc04-cf1b-4dbe-8ceb-64211b06c865 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-089bbc04-cf1b-4dbe-8ceb-64211b06c865');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ],
      "source": [
        "cleanerFn(comments)\n",
        "comments.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "3b94f1bb",
      "metadata": {
        "id": "3b94f1bb"
      },
      "source": [
        "### Natural Language Processing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "84bdefff",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "84bdefff",
        "outputId": "57178d84-36d0-4cbf-9b3e-f5bde27a0835"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data] Downloading package omw-1.4 to /root/nltk_data...\n"
          ]
        }
      ],
      "source": [
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')\n",
        "nltk.download('wordnet')\n",
        "nltk.download('omw-1.4')\n",
        "\n",
        "from nltk.tokenize import sent_tokenize, word_tokenize\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem.porter import *\n",
        "from nltk.stem import PorterStemmer\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f01daee4",
      "metadata": {
        "id": "f01daee4"
      },
      "outputs": [],
      "source": [
        "sw = stopwords.words('english')\n",
        "ps = PorterStemmer()\n",
        "lemmatizer = nltk.stem.WordNetLemmatizer()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "3187d77e",
      "metadata": {
        "id": "3187d77e"
      },
      "source": [
        "#### Tokenization, Remove Stop Words, Lemmatization & Stemming"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b6de5bf2",
      "metadata": {
        "id": "b6de5bf2"
      },
      "outputs": [],
      "source": [
        "def nlpFunction(DF):\n",
        "    DF['com_token'] = DF['comment'].str.lower().str.split()\n",
        "    DF['com_remv'] = DF['com_token'].apply(lambda x: [y for y in x if y not in sw]) #remove stop words\n",
        "    DF[\"com_lemma\"] = DF['com_remv'].apply(lambda x : [lemmatizer.lemmatize(y) for y in x]) # lemmatization\n",
        "    DF['com_stem'] = DF['com_lemma'].apply(lambda x : [ps.stem(y) for y in x]) # stemming\n",
        "    DF[\"com_tok_str\"] = DF[\"com_stem\"].apply(', '.join)\n",
        "    DF[\"com_full\"] = DF[\"com_remv\"].apply(' '.join)\n",
        "    return DF\n",
        "    \n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "7f2ac4be",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 406
        },
        "id": "7f2ac4be",
        "outputId": "beb5b61c-c5fa-4159-ab31-6c0c956ccc68"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   label                                            comment  \\\n",
              "0   -1.0  Everyone knows brand s papers from  But  No on...   \n",
              "1    0.0         Your paper cut balance is                    \n",
              "2    1.0  OH SHIT WHEN I SAW THIS ON MY FRONT PAGE      ...   \n",
              "3    1.0                          Blowing my mind yet again   \n",
              "4    0.0               Should have gone with Dunder Mifflin   \n",
              "\n",
              "                                           com_token  \\\n",
              "0  [everyone, knows, brand, s, papers, from, but,...   \n",
              "1                    [your, paper, cut, balance, is]   \n",
              "2  [oh, shit, when, i, saw, this, on, my, front, ...   \n",
              "3                    [blowing, my, mind, yet, again]   \n",
              "4        [should, have, gone, with, dunder, mifflin]   \n",
              "\n",
              "                                            com_remv  \\\n",
              "0  [everyone, knows, brand, papers, one, knows, w...   \n",
              "1                              [paper, cut, balance]   \n",
              "2           [oh, shit, saw, front, page, love, song]   \n",
              "3                               [blowing, mind, yet]   \n",
              "4                            [gone, dunder, mifflin]   \n",
              "\n",
              "                                           com_lemma  \\\n",
              "0  [everyone, know, brand, paper, one, know, welf...   \n",
              "1                              [paper, cut, balance]   \n",
              "2           [oh, shit, saw, front, page, love, song]   \n",
              "3                               [blowing, mind, yet]   \n",
              "4                            [gone, dunder, mifflin]   \n",
              "\n",
              "                                            com_stem  \\\n",
              "0  [everyon, know, brand, paper, one, know, welfa...   \n",
              "1                               [paper, cut, balanc]   \n",
              "2           [oh, shit, saw, front, page, love, song]   \n",
              "3                                  [blow, mind, yet]   \n",
              "4                            [gone, dunder, mifflin]   \n",
              "\n",
              "                                         com_tok_str  \\\n",
              "0  everyon, know, brand, paper, one, know, welfar...   \n",
              "1                                 paper, cut, balanc   \n",
              "2             oh, shit, saw, front, page, love, song   \n",
              "3                                    blow, mind, yet   \n",
              "4                              gone, dunder, mifflin   \n",
              "\n",
              "                                            com_full  \n",
              "0  everyone knows brand papers one knows welfare ...  \n",
              "1                                  paper cut balance  \n",
              "2                   oh shit saw front page love song  \n",
              "3                                   blowing mind yet  \n",
              "4                                gone dunder mifflin  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-cd0fb872-7cfd-47c4-b606-7daad0c78413\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>comment</th>\n",
              "      <th>com_token</th>\n",
              "      <th>com_remv</th>\n",
              "      <th>com_lemma</th>\n",
              "      <th>com_stem</th>\n",
              "      <th>com_tok_str</th>\n",
              "      <th>com_full</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-1.0</td>\n",
              "      <td>Everyone knows brand s papers from  But  No on...</td>\n",
              "      <td>[everyone, knows, brand, s, papers, from, but,...</td>\n",
              "      <td>[everyone, knows, brand, papers, one, knows, w...</td>\n",
              "      <td>[everyone, know, brand, paper, one, know, welf...</td>\n",
              "      <td>[everyon, know, brand, paper, one, know, welfa...</td>\n",
              "      <td>everyon, know, brand, paper, one, know, welfar...</td>\n",
              "      <td>everyone knows brand papers one knows welfare ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.0</td>\n",
              "      <td>Your paper cut balance is</td>\n",
              "      <td>[your, paper, cut, balance, is]</td>\n",
              "      <td>[paper, cut, balance]</td>\n",
              "      <td>[paper, cut, balance]</td>\n",
              "      <td>[paper, cut, balanc]</td>\n",
              "      <td>paper, cut, balanc</td>\n",
              "      <td>paper cut balance</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.0</td>\n",
              "      <td>OH SHIT WHEN I SAW THIS ON MY FRONT PAGE      ...</td>\n",
              "      <td>[oh, shit, when, i, saw, this, on, my, front, ...</td>\n",
              "      <td>[oh, shit, saw, front, page, love, song]</td>\n",
              "      <td>[oh, shit, saw, front, page, love, song]</td>\n",
              "      <td>[oh, shit, saw, front, page, love, song]</td>\n",
              "      <td>oh, shit, saw, front, page, love, song</td>\n",
              "      <td>oh shit saw front page love song</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.0</td>\n",
              "      <td>Blowing my mind yet again</td>\n",
              "      <td>[blowing, my, mind, yet, again]</td>\n",
              "      <td>[blowing, mind, yet]</td>\n",
              "      <td>[blowing, mind, yet]</td>\n",
              "      <td>[blow, mind, yet]</td>\n",
              "      <td>blow, mind, yet</td>\n",
              "      <td>blowing mind yet</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.0</td>\n",
              "      <td>Should have gone with Dunder Mifflin</td>\n",
              "      <td>[should, have, gone, with, dunder, mifflin]</td>\n",
              "      <td>[gone, dunder, mifflin]</td>\n",
              "      <td>[gone, dunder, mifflin]</td>\n",
              "      <td>[gone, dunder, mifflin]</td>\n",
              "      <td>gone, dunder, mifflin</td>\n",
              "      <td>gone dunder mifflin</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-cd0fb872-7cfd-47c4-b606-7daad0c78413')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-cd0fb872-7cfd-47c4-b606-7daad0c78413 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-cd0fb872-7cfd-47c4-b606-7daad0c78413');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ],
      "source": [
        "comments = nlpFunction(comments)\n",
        "comments.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "249f017d",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "249f017d",
        "outputId": "fb863230-a515-4b24-d93d-aafe61e573c4"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   label                                           com_full\n",
              "0   -1.0  everyone knows brand papers one knows welfare ...\n",
              "1    0.0                                  paper cut balance\n",
              "2    1.0                   oh shit saw front page love song\n",
              "3    1.0                                   blowing mind yet\n",
              "4    0.0                                gone dunder mifflin"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6dcc1d00-825f-44f8-ac40-9b887e8710cf\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>com_full</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-1.0</td>\n",
              "      <td>everyone knows brand papers one knows welfare ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.0</td>\n",
              "      <td>paper cut balance</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.0</td>\n",
              "      <td>oh shit saw front page love song</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.0</td>\n",
              "      <td>blowing mind yet</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.0</td>\n",
              "      <td>gone dunder mifflin</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6dcc1d00-825f-44f8-ac40-9b887e8710cf')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6dcc1d00-825f-44f8-ac40-9b887e8710cf button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6dcc1d00-825f-44f8-ac40-9b887e8710cf');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ],
      "source": [
        "def drop_cols_after_nlp(comments):\n",
        "    comments = comments.drop(columns = ['comment', 'com_token', 'com_remv', 'com_lemma', 'com_stem', 'com_tok_str'], axis = 1)\n",
        "    return comments\n",
        "comments = drop_cols_after_nlp(comments)\n",
        "comments.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "2a2ddfad",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "2a2ddfad",
        "outputId": "75873d66-ce74-4671-c252-1d9784188ced"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   label                                            comment\n",
              "0   -1.0  everyone knows brand papers one knows welfare ...\n",
              "1    0.0                                  paper cut balance\n",
              "2    1.0                   oh shit saw front page love song\n",
              "3    1.0                                   blowing mind yet\n",
              "4    0.0                                gone dunder mifflin"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a50d5cde-f9c8-42b8-915a-fa0afb328f8c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>comment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-1.0</td>\n",
              "      <td>everyone knows brand papers one knows welfare ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.0</td>\n",
              "      <td>paper cut balance</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.0</td>\n",
              "      <td>oh shit saw front page love song</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.0</td>\n",
              "      <td>blowing mind yet</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.0</td>\n",
              "      <td>gone dunder mifflin</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a50d5cde-f9c8-42b8-915a-fa0afb328f8c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a50d5cde-f9c8-42b8-915a-fa0afb328f8c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a50d5cde-f9c8-42b8-915a-fa0afb328f8c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ],
      "source": [
        "comments.rename(columns = {'com_full': 'comment'}, inplace=True)\n",
        "comments.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "90fe2cdc",
      "metadata": {
        "id": "90fe2cdc"
      },
      "outputs": [],
      "source": [
        "def remove_missing_vals(comments): \n",
        "    comments['comment'] = comments['comment'].str.strip()\n",
        "    comments = comments[comments.comment != 'nan'] # remove nan values from data\n",
        "    comments = comments[comments.comment != '']\n",
        "    \n",
        "remove_missing_vals(comments)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "923090e8",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "923090e8",
        "outputId": "b7bfeb08-434b-4058-c62b-7bb47e348768"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   label                                            comment\n",
              "0   -1.0  everyone knows brand papers one knows welfare ...\n",
              "1    0.0                                  paper cut balance\n",
              "2    1.0                   oh shit saw front page love song\n",
              "3    1.0                                   blowing mind yet\n",
              "4    0.0                                gone dunder mifflin"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a895cf87-9fdf-4460-8a8b-74bbb8db8406\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>comment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-1.0</td>\n",
              "      <td>everyone knows brand papers one knows welfare ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.0</td>\n",
              "      <td>paper cut balance</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.0</td>\n",
              "      <td>oh shit saw front page love song</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.0</td>\n",
              "      <td>blowing mind yet</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.0</td>\n",
              "      <td>gone dunder mifflin</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a895cf87-9fdf-4460-8a8b-74bbb8db8406')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a895cf87-9fdf-4460-8a8b-74bbb8db8406 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a895cf87-9fdf-4460-8a8b-74bbb8db8406');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ],
      "source": [
        "comments.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "cbcc2ae0",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cbcc2ae0",
        "outputId": "34c960f5-6920-429a-afc9-8b34b0391fa7"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2355"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ],
      "source": [
        "comments['label'].isna().sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b3bc396d",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b3bc396d",
        "outputId": "1b20499e-8c6d-41cb-ea3d-8a39609be01e"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ],
      "source": [
        "comments = comments[comments['label'].notna()]\n",
        "comments['label'].isna().sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "318c8c8c",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "318c8c8c",
        "outputId": "e914b724-3cf6-40a8-8e6a-41dde30b3540"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "14830"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ],
      "source": [
        "len(comments)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f0531a4d",
      "metadata": {
        "id": "f0531a4d"
      },
      "outputs": [],
      "source": [
        "X = comments['comment']\n",
        "y = comments.label"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "258b7c33",
      "metadata": {
        "id": "258b7c33"
      },
      "outputs": [],
      "source": [
        "# split X and y into training and testing sets\n",
        "from sklearn.model_selection import train_test_split\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=53, test_size=0.25)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "5691e17d",
      "metadata": {
        "id": "5691e17d"
      },
      "source": [
        "### Vectorize the tweets\n",
        "<p>We have the training and testing data all set up, but we need to create vectorized representations of the tweets in order to apply machine learning.</p>\n",
        "<p>To do so, we will utilize the <code>CountVectorizer</code> and <code>TfidfVectorizer</code> classes which we will first need to fit to the data.</p>\n",
        "<p>Once this is complete, we can start modeling with the new vectorized tweets!</p>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "91258b43",
      "metadata": {
        "id": "91258b43"
      },
      "outputs": [],
      "source": [
        "# Initialize count vectorizer\n",
        "count_vectorizer = CountVectorizer(stop_words='english', \n",
        "                                   min_df=0.05, max_df=0.9) #max limit se jada then eliminate stopword ka token eliminate\n",
        "\n",
        "# Create count train and test variables\n",
        "count_train = count_vectorizer.fit_transform(X_train)\n",
        "count_test = count_vectorizer.transform(X_test)\n",
        "\n",
        "# Initialize tfidf vectorizer\n",
        "tfidf_vectorizer = TfidfVectorizer(stop_words='english', \n",
        "                                   min_df=0.05, max_df=0.9)\n",
        "\n",
        "# Create tfidf train and test variables\n",
        "tfidf_train = tfidf_vectorizer.fit_transform(X_train)\n",
        "tfidf_test = tfidf_vectorizer.transform(X_test)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "b48a0756",
      "metadata": {
        "id": "b48a0756"
      },
      "source": [
        "## Model Building"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ec3c1ad9",
      "metadata": {
        "id": "ec3c1ad9"
      },
      "outputs": [],
      "source": [
        "# Set seed for reproducibility\n",
        "import random; random.seed(5)\n",
        "\n",
        "# Import all we need from sklearn\n",
        "from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.svm import LinearSVC\n",
        "from sklearn import metrics"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "9d7b0c7d",
      "metadata": {
        "id": "9d7b0c7d"
      },
      "source": [
        "### Multinomial Naive-Bayes Model\n",
        "Training a multinomial naive Bayes model\n",
        "<p>Now that we have the data in vectorized form, we can train the first model. Investigate using the Multinomial Naive Bayes model with both the <code>CountVectorizer</code> and <code>TfidfVectorizer</code> data. Which do will perform better? How come?</p>\n",
        "<p>To assess the accuracies, we will print the test sets accuracy scores for both models.</p>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "0dd136e6",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0dd136e6",
        "outputId": "b13c6c99-e798-410c-8ec8-b4be86824618"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "NaiveBayes Tfidf Score:  0.7909924487594391\n",
            "NaiveBayes Count Score:  0.7831715210355987\n"
          ]
        }
      ],
      "source": [
        "# Create a MulitnomialNB model\n",
        "tfidf_nb = MultinomialNB()\n",
        "tfidf_nb.fit(tfidf_train,y_train)\n",
        "# Run predict on your TF-IDF test data to get your predictions\n",
        "tfidf_nb_pred = tfidf_nb.predict(tfidf_test)\n",
        "\n",
        "# Calculate the accuracy of your predictions\n",
        "tfidf_nb_score = metrics.accuracy_score(y_test,tfidf_nb_pred)\n",
        "\n",
        "# Create a MulitnomialNB model\n",
        "count_nb = MultinomialNB()\n",
        "count_nb.fit(count_train,y_train)\n",
        "\n",
        "# Run predict on your count test data to get your predictions\n",
        "count_nb_pred = count_nb.predict(count_test)\n",
        "\n",
        "# Calculate the accuracy of your predictions\n",
        "count_nb_score = metrics.accuracy_score(count_nb_pred,y_test)\n",
        "\n",
        "print('NaiveBayes Tfidf Score: ', tfidf_nb_score)\n",
        "print('NaiveBayes Count Score: ', count_nb_score)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "a4e8bef1",
      "metadata": {
        "id": "a4e8bef1"
      },
      "source": [
        "### Logistic Regression"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "dfdfb23b",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dfdfb23b",
        "outputId": "f66ef1eb-3c95-404d-f437-634f8b123b57"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Logistic Regression accuracy is (for Tfidf) : 0.7880258899676376\n"
          ]
        }
      ],
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "lr_model = LogisticRegression()\n",
        "lr_model.fit(tfidf_train,y_train)\n",
        "accuracy_lr = lr_model.score(tfidf_test,y_test)\n",
        "print(\"Logistic Regression accuracy is (for Tfidf) :\",accuracy_lr)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "40a57742",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "40a57742",
        "outputId": "b4c94a88-b76b-4446-fc21-63d0560ae280"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Logistic Regression accuracy is (for Count) : 0.7877562028047465\n"
          ]
        }
      ],
      "source": [
        "lr_model = LogisticRegression()\n",
        "lr_model.fit(count_train,y_train)\n",
        "accuracy_lr = lr_model.score(count_test,y_test)\n",
        "print(\"Logistic Regression accuracy is (for Count) :\",accuracy_lr)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "a91191ff",
      "metadata": {
        "id": "a91191ff"
      },
      "source": [
        "### SVC"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "1001a01a",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1001a01a",
        "outputId": "5769ca4e-eead-4692-f956-98c3c5de76f4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "LinearSVC Score (for tfidf):   0.792\n"
          ]
        }
      ],
      "source": [
        "# Create a SVM model\n",
        "from sklearn import svm\n",
        "tfidf_svc = svm.SVC(kernel='linear', C=1)\n",
        "\n",
        "tfidf_svc.fit(tfidf_train,y_train)\n",
        "# Run predict on your tfidf test data to get your predictions\n",
        "tfidf_svc_pred = tfidf_svc.predict(tfidf_test)\n",
        "\n",
        "# Calculate your accuracy using the metrics module\n",
        "tfidf_svc_score = metrics.accuracy_score(y_test,tfidf_svc_pred)\n",
        "\n",
        "print(\"LinearSVC Score (for tfidf):   %0.3f\" % tfidf_svc_score)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "07f3f65c",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "07f3f65c",
        "outputId": "ee46fdfc-1dc8-46c0-be96-1b62cee1830c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "LinearSVC Score (for Count):   0.792\n"
          ]
        }
      ],
      "source": [
        "count_svc = svm.SVC(kernel='linear', C=1)\n",
        "\n",
        "count_svc.fit(count_train,y_train)\n",
        "# Run predict on your count test data to get your predictions\n",
        "count_svc_pred = count_svc.predict(count_test)\n",
        "\n",
        "# Calculate your accuracy using the metrics module\n",
        "count_svc_score = metrics.accuracy_score(y_test,count_svc_pred)\n",
        "\n",
        "print(\"LinearSVC Score (for Count):   %0.3f\" % tfidf_svc_score)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "b0f2f2ea",
      "metadata": {
        "id": "b0f2f2ea"
      },
      "source": [
        "### Desicion Tree"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "id": "9ff217a6",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        },
        "id": "9ff217a6",
        "outputId": "bb375d9f-fa30-4bc6-8ba6-31c6d94cddeb"
      },
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-1-d5c5d1b210e4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtree\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDecisionTreeClassifier\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mdt_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDecisionTreeClassifier\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mdt_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtfidf_train\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0maccuracy_dt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdt_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtfidf_test\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Decision Tree accuracy is (for Tfidf):\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0maccuracy_dt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'tfidf_train' is not defined"
          ]
        }
      ],
      "source": [
        "from sklearn.tree import DecisionTreeClassifier\n",
        "dt_model = DecisionTreeClassifier()\n",
        "dt_model.fit(tfidf_train,y_train)\n",
        "accuracy_dt = dt_model.score(tfidf_test,y_test)\n",
        "print(\"Decision Tree accuracy is (for Tfidf):\",accuracy_dt)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "47e5daef",
      "metadata": {
        "id": "47e5daef"
      },
      "outputs": [],
      "source": [
        "dt_model = DecisionTreeClassifier()\n",
        "dt_model.fit(count_train,y_train)\n",
        "accuracy_dt = dt_model.score(count_test,y_test)\n",
        "print(\"Decision Tree accuracy is (for Count):\",accuracy_dt)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "64123c87",
      "metadata": {
        "id": "64123c87"
      },
      "source": [
        "### Random Forest"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6c97b248",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6c97b248",
        "outputId": "851aa72e-183b-4daa-f98d-de3ab6bf5567"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Random Forest accuracy for 5 trees is (Tfidf): 0.7977346278317152\n"
          ]
        }
      ],
      "source": [
        "from sklearn.ensemble import RandomForestClassifier\n",
        "rf_model_initial = RandomForestClassifier(n_estimators = 5, random_state = 1)\n",
        "rf_model_initial.fit(tfidf_train,y_train)\n",
        "print(\"Random Forest accuracy for 5 trees is (Tfidf):\",rf_model_initial.score(tfidf_test,y_test))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3a4fd33b",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3a4fd33b",
        "outputId": "af7c9f95-82d4-48dd-d090-c7c28cef0d33"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Random Forest accuracy for 5 trees is (Count): 0.7974649406688241\n"
          ]
        }
      ],
      "source": [
        "rf_model_initial = RandomForestClassifier(n_estimators = 5, random_state = 1)\n",
        "rf_model_initial.fit(count_train,y_train)\n",
        "print(\"Random Forest accuracy for 5 trees is (Count):\",rf_model_initial.score(count_test,y_test))"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "940e8d21",
      "metadata": {
        "id": "940e8d21"
      },
      "source": [
        "# Predicting Sentiment For YouTube video\n",
        "\n",
        "**Steps:**\n",
        "1. Go to the YouTube video of your choice and download the comments for that YouTube video and add it into the Comments.csv file.\n",
        "3. Then execute the below files.\n",
        "\n",
        "## Reading Data\n",
        "\n",
        "### Reading Testing YouTube Video Comments\n",
        "\n",
        "Comments.csv files has comments of youtube video"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5b821e5a",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "5b821e5a",
        "outputId": "121db076-397e-4475-b7db-8e5566275b9d"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                             comment\n",
              "0  What do YOU think to the current state of Fold...\n",
              "1  Well, finally someone who can compete with Sam...\n",
              "2  I wanna see them attempt something like the Z-...\n",
              "3  4:57 \"And then actually coming with the charge...\n",
              "4  Personally, for me this was one of, if not the..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e832ecc3-d435-41d3-afb7-fefdd924fa13\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>comment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>What do YOU think to the current state of Fold...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Well, finally someone who can compete with Sam...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>I wanna see them attempt something like the Z-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4:57 \"And then actually coming with the charge...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Personally, for me this was one of, if not the...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e832ecc3-d435-41d3-afb7-fefdd924fa13')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e832ecc3-d435-41d3-afb7-fefdd924fa13 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e832ecc3-d435-41d3-afb7-fefdd924fa13');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ],
      "source": [
        "prediction_comments = pd.read_csv('Comments.csv', delimiter=\",\", encoding='utf-8', engine='python')\n",
        "prediction_comments = prediction_comments.iloc[:,:1]\n",
        "prediction_comments.columns=['comment']\n",
        "prediction_comments.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "1c022176",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "1c022176",
        "outputId": "8e8a1b86-0b8b-4f8a-bc93-e4873f666642"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                             comment\n",
              "0  What do YOU think to the current state of Fold...\n",
              "1  Well, finally someone who can compete with Sam...\n",
              "2  I wanna see them attempt something like the Z-...\n",
              "3  4:57 \"And then actually coming with the charge...\n",
              "4  Personally, for me this was one of, if not the..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-46c9a477-e3bb-429a-b692-7397428af246\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>comment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>What do YOU think to the current state of Fold...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Well, finally someone who can compete with Sam...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>I wanna see them attempt something like the Z-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4:57 \"And then actually coming with the charge...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Personally, for me this was one of, if not the...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-46c9a477-e3bb-429a-b692-7397428af246')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-46c9a477-e3bb-429a-b692-7397428af246 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-46c9a477-e3bb-429a-b692-7397428af246');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ],
      "source": [
        "# Lets use SVC to predict on our youtube video comments\n",
        "prediction_comments.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b7398f78",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b7398f78",
        "outputId": "4108caf1-52aa-4224-ecf4-5fee6645ee86"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1001"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ],
      "source": [
        "len(prediction_comments['comment'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "1cf54e12",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "1cf54e12",
        "outputId": "aae0d422-a3af-4bea-d48f-d5c74e4c56e9"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                             comment\n",
              "0  think current state foldable phones check tesl...\n",
              "1  well finally someone compete samsung market co...\n",
              "2  wanna see attempt something like z flip someth...\n",
              "3  actually coming charger respect xiaomi getting...\n",
              "4  personally one best video ever made simple alw..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e9e44de0-bd06-4c4b-a6fd-cbae7029dfb2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>comment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>think current state foldable phones check tesl...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>well finally someone compete samsung market co...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>wanna see attempt something like z flip someth...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>actually coming charger respect xiaomi getting...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>personally one best video ever made simple alw...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e9e44de0-bd06-4c4b-a6fd-cbae7029dfb2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e9e44de0-bd06-4c4b-a6fd-cbae7029dfb2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e9e44de0-bd06-4c4b-a6fd-cbae7029dfb2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 45
        }
      ],
      "source": [
        "convert_to_string(prediction_comments)\n",
        "cleanerFn(prediction_comments)\n",
        "prediction_comments = nlpFunction(prediction_comments)\n",
        "prediction_comments = drop_cols_after_nlp(prediction_comments)\n",
        "prediction_comments.rename(columns = {'com_full': 'comment'}, inplace=True)\n",
        "remove_missing_vals(prediction_comments)\n",
        "prediction_comments.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "94dddc7f",
      "metadata": {
        "id": "94dddc7f"
      },
      "outputs": [],
      "source": [
        "tfidf_pred = tfidf_vectorizer.transform(prediction_comments['comment'])\n",
        "tfidf_svc_pred = tfidf_svc.predict(tfidf_pred)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6d1d0f70",
      "metadata": {
        "id": "6d1d0f70"
      },
      "outputs": [],
      "source": [
        "neutral = (tfidf_svc_pred == 0.0).sum()\n",
        "positive = (tfidf_svc_pred == 1.0).sum()\n",
        "negative = (tfidf_svc_pred < 0).sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "7a5251c8",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7a5251c8",
        "outputId": "4b943744-3108-48ea-e0a4-a7db5bfa3e05"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "833 161 7\n"
          ]
        }
      ],
      "source": [
        "print(neutral, positive, negative)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "9f6298d4",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9f6298d4",
        "outputId": "58b53cb7-fe7c-49c2-932c-6156483437ca"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Good video\n"
          ]
        }
      ],
      "source": [
        "print(\"Good video\" if positive > negative else \"Bad video\")"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.12"
    },
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "b0f2f2ea",
        "64123c87"
      ]
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}