Experiment_3_SentimentNumerical_Model.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Experiment_3_SentimentNumerical_Model.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMREg0+1mtu8vJ3cMhXIZBq",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/thejesht/machine_learning_projects/blob/main/Experiment_3_SentimentNumerical_Model.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1CVwcAbq_Hy1"
      },
      "source": [
        "#importing libraries required for analysis\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import re\n",
        "from bs4 import BeautifulSoup\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 135,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SKuP9IVa_gN-",
        "outputId": "e530e83d-4aaa-4e16-c52a-3e353f974425"
      },
      "source": [
        "#importing tweet dataset acquired using tweet scrapper built\n",
        "df=pd.read_csv('20210815_btc_tweets.csv')\n",
        "#exploring dimensions of imported dataset\n",
        "df.shape "
      ],
      "execution_count": 136,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(28854, 36)"
            ]
          },
          "metadata": {},
          "execution_count": 136
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 564
        },
        "id": "UER0J65BAUuW",
        "outputId": "3a76c5ca-edd3-4390-cbf7-a0afc8080027"
      },
      "source": [
        "#visualising imported dataset\n",
        "df.head()"
      ],
      "execution_count": 137,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>conversation_id</th>\n",
              "      <th>created_at</th>\n",
              "      <th>date</th>\n",
              "      <th>time</th>\n",
              "      <th>timezone</th>\n",
              "      <th>user_id</th>\n",
              "      <th>username</th>\n",
              "      <th>name</th>\n",
              "      <th>place</th>\n",
              "      <th>tweet</th>\n",
              "      <th>language</th>\n",
              "      <th>mentions</th>\n",
              "      <th>urls</th>\n",
              "      <th>photos</th>\n",
              "      <th>replies_count</th>\n",
              "      <th>retweets_count</th>\n",
              "      <th>likes_count</th>\n",
              "      <th>hashtags</th>\n",
              "      <th>cashtags</th>\n",
              "      <th>link</th>\n",
              "      <th>retweet</th>\n",
              "      <th>quote_url</th>\n",
              "      <th>video</th>\n",
              "      <th>thumbnail</th>\n",
              "      <th>near</th>\n",
              "      <th>geo</th>\n",
              "      <th>source</th>\n",
              "      <th>user_rt_id</th>\n",
              "      <th>user_rt</th>\n",
              "      <th>retweet_id</th>\n",
              "      <th>reply_to</th>\n",
              "      <th>retweet_date</th>\n",
              "      <th>translate</th>\n",
              "      <th>trans_src</th>\n",
              "      <th>trans_dest</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1413396249546563586</td>\n",
              "      <td>1413355442475085825</td>\n",
              "      <td>2021-07-09 08:15:00 GMT Daylight Time</td>\n",
              "      <td>2021-07-09</td>\n",
              "      <td>08:15:00</td>\n",
              "      <td>100</td>\n",
              "      <td>44196397</td>\n",
              "      <td>elonmusk</td>\n",
              "      <td>Elon Musk</td>\n",
              "      <td>NaN</td>\n",
              "      <td>@MattWallace888 BTC &amp;amp; ETH are pursuing a m...</td>\n",
              "      <td>en</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>7005</td>\n",
              "      <td>6712</td>\n",
              "      <td>27392</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>https://twitter.com/elonmusk/status/1413396249...</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[{'screen_name': 'MattWallace888', 'name': 'Ma...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1404132183254523905</td>\n",
              "      <td>1404045922674302982</td>\n",
              "      <td>2021-06-13 18:42:54 GMT Daylight Time</td>\n",
              "      <td>2021-06-13</td>\n",
              "      <td>18:42:54</td>\n",
              "      <td>100</td>\n",
              "      <td>44196397</td>\n",
              "      <td>elonmusk</td>\n",
              "      <td>Elon Musk</td>\n",
              "      <td>NaN</td>\n",
              "      <td>@Cointelegraph This is inaccurate. Tesla only ...</td>\n",
              "      <td>en</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>10126</td>\n",
              "      <td>10893</td>\n",
              "      <td>54738</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>https://twitter.com/elonmusk/status/1404132183...</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[{'screen_name': 'Cointelegraph', 'name': 'Coi...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1401091921746006017</td>\n",
              "      <td>1400560049358094340</td>\n",
              "      <td>2021-06-05 09:21:59 GMT Daylight Time</td>\n",
              "      <td>2021-06-05</td>\n",
              "      <td>09:21:59</td>\n",
              "      <td>100</td>\n",
              "      <td>44196397</td>\n",
              "      <td>elonmusk</td>\n",
              "      <td>Elon Musk</td>\n",
              "      <td>NaN</td>\n",
              "      <td>@lexfridman @VitalikButerin @ethereum @Bitcoin...</td>\n",
              "      <td>en</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>2065</td>\n",
              "      <td>3309</td>\n",
              "      <td>24705</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>https://twitter.com/elonmusk/status/1401091921...</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[{'screen_name': 'lexfridman', 'name': 'Lex Fr...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1397626694609956864</td>\n",
              "      <td>1397625459437826049</td>\n",
              "      <td>2021-05-26 19:52:25 GMT Daylight Time</td>\n",
              "      <td>2021-05-26</td>\n",
              "      <td>19:52:25</td>\n",
              "      <td>100</td>\n",
              "      <td>44196397</td>\n",
              "      <td>elonmusk</td>\n",
              "      <td>Elon Musk</td>\n",
              "      <td>NaN</td>\n",
              "      <td>@BTC_Archive 🤣🤣</td>\n",
              "      <td>und</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>921</td>\n",
              "      <td>330</td>\n",
              "      <td>8609</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>https://twitter.com/elonmusk/status/1397626694...</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[{'screen_name': 'BTC_Archive', 'name': 'Bitco...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1394170030741413888</td>\n",
              "      <td>1394051036462071808</td>\n",
              "      <td>2021-05-17 06:56:52 GMT Daylight Time</td>\n",
              "      <td>2021-05-17</td>\n",
              "      <td>06:56:52</td>\n",
              "      <td>100</td>\n",
              "      <td>44196397</td>\n",
              "      <td>elonmusk</td>\n",
              "      <td>Elon Musk</td>\n",
              "      <td>NaN</td>\n",
              "      <td>@BTC_Archive To clarify speculation, Tesla has...</td>\n",
              "      <td>en</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>18117</td>\n",
              "      <td>17855</td>\n",
              "      <td>101526</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>https://twitter.com/elonmusk/status/1394170030...</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[{'screen_name': 'BTC_Archive', 'name': 'Bitco...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                    id      conversation_id  ... trans_src trans_dest\n",
              "0  1413396249546563586  1413355442475085825  ...       NaN        NaN\n",
              "1  1404132183254523905  1404045922674302982  ...       NaN        NaN\n",
              "2  1401091921746006017  1400560049358094340  ...       NaN        NaN\n",
              "3  1397626694609956864  1397625459437826049  ...       NaN        NaN\n",
              "4  1394170030741413888  1394051036462071808  ...       NaN        NaN\n",
              "\n",
              "[5 rows x 36 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 137
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IkKO91AsAZtY",
        "outputId": "acdad121-cb7b-41c3-b61f-e1e4a1f4369a"
      },
      "source": [
        "#exploring for null values in imported dataset\n",
        "df.isnull().sum()"
      ],
      "execution_count": 138,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "id                     0\n",
              "conversation_id        0\n",
              "created_at             0\n",
              "date                   0\n",
              "time                   0\n",
              "timezone               0\n",
              "user_id                0\n",
              "username               0\n",
              "name                   0\n",
              "place              28854\n",
              "tweet                  0\n",
              "language               0\n",
              "mentions               0\n",
              "urls                   0\n",
              "photos                 0\n",
              "replies_count          0\n",
              "retweets_count         0\n",
              "likes_count            0\n",
              "hashtags               0\n",
              "cashtags               0\n",
              "link                   0\n",
              "retweet                0\n",
              "quote_url          26172\n",
              "video                  0\n",
              "thumbnail          17393\n",
              "near               28854\n",
              "geo                28854\n",
              "source             28854\n",
              "user_rt_id         28854\n",
              "user_rt            28854\n",
              "retweet_id         28854\n",
              "reply_to               0\n",
              "retweet_date       28854\n",
              "translate          28854\n",
              "trans_src          28854\n",
              "trans_dest         28854\n",
              "dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 138
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "oKlNghNmAmb-",
        "outputId": "827f1ef0-f3c2-4e7d-d4bd-016fbd4720c7"
      },
      "source": [
        "#filtering fields from dataset which shall be used for further analysis \n",
        "df= df.filter(items=['date', 'time','username','tweet','replies_count','retweets_count','likes_count'])\n",
        "df.head()"
      ],
      "execution_count": 139,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>time</th>\n",
              "      <th>username</th>\n",
              "      <th>tweet</th>\n",
              "      <th>replies_count</th>\n",
              "      <th>retweets_count</th>\n",
              "      <th>likes_count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2021-07-09</td>\n",
              "      <td>08:15:00</td>\n",
              "      <td>elonmusk</td>\n",
              "      <td>@MattWallace888 BTC &amp;amp; ETH are pursuing a m...</td>\n",
              "      <td>7005</td>\n",
              "      <td>6712</td>\n",
              "      <td>27392</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2021-06-13</td>\n",
              "      <td>18:42:54</td>\n",
              "      <td>elonmusk</td>\n",
              "      <td>@Cointelegraph This is inaccurate. Tesla only ...</td>\n",
              "      <td>10126</td>\n",
              "      <td>10893</td>\n",
              "      <td>54738</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2021-06-05</td>\n",
              "      <td>09:21:59</td>\n",
              "      <td>elonmusk</td>\n",
              "      <td>@lexfridman @VitalikButerin @ethereum @Bitcoin...</td>\n",
              "      <td>2065</td>\n",
              "      <td>3309</td>\n",
              "      <td>24705</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2021-05-26</td>\n",
              "      <td>19:52:25</td>\n",
              "      <td>elonmusk</td>\n",
              "      <td>@BTC_Archive 🤣🤣</td>\n",
              "      <td>921</td>\n",
              "      <td>330</td>\n",
              "      <td>8609</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2021-05-17</td>\n",
              "      <td>06:56:52</td>\n",
              "      <td>elonmusk</td>\n",
              "      <td>@BTC_Archive To clarify speculation, Tesla has...</td>\n",
              "      <td>18117</td>\n",
              "      <td>17855</td>\n",
              "      <td>101526</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "         date      time  username  ... replies_count  retweets_count  likes_count\n",
              "0  2021-07-09  08:15:00  elonmusk  ...          7005            6712        27392\n",
              "1  2021-06-13  18:42:54  elonmusk  ...         10126           10893        54738\n",
              "2  2021-06-05  09:21:59  elonmusk  ...          2065            3309        24705\n",
              "3  2021-05-26  19:52:25  elonmusk  ...           921             330         8609\n",
              "4  2021-05-17  06:56:52  elonmusk  ...         18117           17855       101526\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 139
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A4ITayIHAzYW",
        "outputId": "ce1a8f7c-93f4-47a7-a634-2d11db36e69b"
      },
      "source": [
        "#exploring null values post filtering of required fields for analysis\n",
        "df.isnull().sum()"
      ],
      "execution_count": 140,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "date              0\n",
              "time              0\n",
              "username          0\n",
              "tweet             0\n",
              "replies_count     0\n",
              "retweets_count    0\n",
              "likes_count       0\n",
              "dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 140
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "2QriXbjaA6hr",
        "outputId": "55e7ba40-51f8-4923-c827-0580faf7ae18"
      },
      "source": [
        "#creating time stamp 'date'\n",
        "df.loc[:,'date'] = pd.to_datetime(df.date.astype(str)+' '+df.time.astype(str))\n",
        "df.drop(columns=['time'], inplace= True)\n",
        "df['date'] = df['date'] - pd.to_timedelta(df['date'].dt.second, unit='s')\n",
        "df.head()"
      ],
      "execution_count": 141,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>username</th>\n",
              "      <th>tweet</th>\n",
              "      <th>replies_count</th>\n",
              "      <th>retweets_count</th>\n",
              "      <th>likes_count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2021-07-09 08:15:00</td>\n",
              "      <td>elonmusk</td>\n",
              "      <td>@MattWallace888 BTC &amp;amp; ETH are pursuing a m...</td>\n",
              "      <td>7005</td>\n",
              "      <td>6712</td>\n",
              "      <td>27392</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2021-06-13 18:42:00</td>\n",
              "      <td>elonmusk</td>\n",
              "      <td>@Cointelegraph This is inaccurate. Tesla only ...</td>\n",
              "      <td>10126</td>\n",
              "      <td>10893</td>\n",
              "      <td>54738</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2021-06-05 09:21:00</td>\n",
              "      <td>elonmusk</td>\n",
              "      <td>@lexfridman @VitalikButerin @ethereum @Bitcoin...</td>\n",
              "      <td>2065</td>\n",
              "      <td>3309</td>\n",
              "      <td>24705</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2021-05-26 19:52:00</td>\n",
              "      <td>elonmusk</td>\n",
              "      <td>@BTC_Archive 🤣🤣</td>\n",
              "      <td>921</td>\n",
              "      <td>330</td>\n",
              "      <td>8609</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2021-05-17 06:56:00</td>\n",
              "      <td>elonmusk</td>\n",
              "      <td>@BTC_Archive To clarify speculation, Tesla has...</td>\n",
              "      <td>18117</td>\n",
              "      <td>17855</td>\n",
              "      <td>101526</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 date  username  ... retweets_count  likes_count\n",
              "0 2021-07-09 08:15:00  elonmusk  ...           6712        27392\n",
              "1 2021-06-13 18:42:00  elonmusk  ...          10893        54738\n",
              "2 2021-06-05 09:21:00  elonmusk  ...           3309        24705\n",
              "3 2021-05-26 19:52:00  elonmusk  ...            330         8609\n",
              "4 2021-05-17 06:56:00  elonmusk  ...          17855       101526\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 141
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "O3W-SAjhBRxc",
        "outputId": "cd05c68a-a7d4-450a-f325-6c1b62721f4d"
      },
      "source": [
        "#explornig length of field:tweet, prior to data wrangling \n",
        "df['pre_clean_len'] = [len(t) for t in df.tweet]\n",
        "df.head()"
      ],
      "execution_count": 142,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>username</th>\n",
              "      <th>tweet</th>\n",
              "      <th>replies_count</th>\n",
              "      <th>retweets_count</th>\n",
              "      <th>likes_count</th>\n",
              "      <th>pre_clean_len</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2021-07-09 08:15:00</td>\n",
              "      <td>elonmusk</td>\n",
              "      <td>@MattWallace888 BTC &amp;amp; ETH are pursuing a m...</td>\n",
              "      <td>7005</td>\n",
              "      <td>6712</td>\n",
              "      <td>27392</td>\n",
              "      <td>303</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2021-06-13 18:42:00</td>\n",
              "      <td>elonmusk</td>\n",
              "      <td>@Cointelegraph This is inaccurate. Tesla only ...</td>\n",
              "      <td>10126</td>\n",
              "      <td>10893</td>\n",
              "      <td>54738</td>\n",
              "      <td>286</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2021-06-05 09:21:00</td>\n",
              "      <td>elonmusk</td>\n",
              "      <td>@lexfridman @VitalikButerin @ethereum @Bitcoin...</td>\n",
              "      <td>2065</td>\n",
              "      <td>3309</td>\n",
              "      <td>24705</td>\n",
              "      <td>148</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2021-05-26 19:52:00</td>\n",
              "      <td>elonmusk</td>\n",
              "      <td>@BTC_Archive 🤣🤣</td>\n",
              "      <td>921</td>\n",
              "      <td>330</td>\n",
              "      <td>8609</td>\n",
              "      <td>15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2021-05-17 06:56:00</td>\n",
              "      <td>elonmusk</td>\n",
              "      <td>@BTC_Archive To clarify speculation, Tesla has...</td>\n",
              "      <td>18117</td>\n",
              "      <td>17855</td>\n",
              "      <td>101526</td>\n",
              "      <td>67</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 date  username  ... likes_count  pre_clean_len\n",
              "0 2021-07-09 08:15:00  elonmusk  ...       27392            303\n",
              "1 2021-06-13 18:42:00  elonmusk  ...       54738            286\n",
              "2 2021-06-05 09:21:00  elonmusk  ...       24705            148\n",
              "3 2021-05-26 19:52:00  elonmusk  ...        8609             15\n",
              "4 2021-05-17 06:56:00  elonmusk  ...      101526             67\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 142
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 316
        },
        "id": "UGV4cxWiBVLj",
        "outputId": "53924fd9-dc15-40fe-e4cf-4b85c26f0455"
      },
      "source": [
        "#plotting distribution of length of tweets prior to data wrangling\n",
        "fig, ax = plt.subplots(figsize=(5, 5))\n",
        "plt.boxplot(df.pre_clean_len)\n",
        "plt.show()"
      ],
      "execution_count": 143,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAAErCAYAAAB981BrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAANmUlEQVR4nO3df6id9X3A8fdHE+tYt3uqbreSBCL04Og/s6Wulg50SpnaMf2jiEXWmIT5j4GOjjTJ/ujuxiBGYa6SIR1TmwznlG5FKbJNorMMppO2W1d13bkDJQmxUvXcWYRAymd/3O/trll+nCTn+Zyce98vuNzn+T7Pfc73wsM7D0+ec09kJpKkGhdMegKStJoYXUkqZHQlqZDRlaRCRleSCq2ZxIsuLCz4yISkVWFmZiaWr3ulK0mFjK4kFTK6WhUGg8GkpyABRleSShldSSpkdCWpkNGVpEJGV5IKGV1JKmR0taJt376d2dlZrr76amZnZ9m+ffukp6RVbqS3AUfEa8C7wE+BY5n5iYi4BHgc2Ai8BtyWme9ERABfBW4G3gPuzMzvjn/q0qlt376dRx55hLm5Oa699lqef/555ubmALjvvvsmOzmtWmdypfsbmXlVZn6ire8EDmRmHzjQ1gFuAvrt6y7gwXFNVjoT+/fvZ25ujm3btnHxxRezbds25ubm2L9//6SnplXsXG4v3ALsa8v7gFuXje/PRS8AvYi4/BxeRzorR48eZcuWLe8b27JlC0ePHp3QjKTR/8pYAv8YEQl8LTP/ApjNzCNt+xvAbFteBxxc9rOH2tgRTsC3Z6ora9eu5d577+WOO+4AFs+1Rx99lLVr13reqVP9fv+k20aN7q9n5uGI+GXgmYj4z+UbMzNbkMc6Oelc3Hnnnezdu5fLLrvsZ/d09+7dy+bNmz3vNDFxpp8GHBFzwE+A3wWuy8wj7fbBP2XmlRHxtbb8WNv/h0v7LR3Dv6erc7V792727NnT2fF37NjBrl27Oju+Vo/j/57uaaMbET8PXJCZ77blZ4A/Bm4A3srMeyJiJ3BJZn45Ij4LbGPx6YVPAg9k5q8tP6bRVbVer8dwOJz0NLQKHR/dUW4vzALfXHwSjDXAX2fm30fES8ATEbEVeB24re3/NIvBnWfxkbHNY5q7JE29M769MA5e6aqaV7qaFD+uR5ImyOhKUiGjK0mFjK4kFTK6klTI6EpSIaMrSYWMriQVMrqSVMjoSlIhoytJhYyuJBUyupJUyOhKUiGjK0mFjK4kFTK6klTI6EpSIaMrSYWMriQVMrqSVMjoSlIhoytJhYyuJBUyupJUyOhKUiGjK0mFjK4kFTK6klTI6EpSIaMrSYWMriQVMrqSVMjoSlIhoytJhYyuJBUyupJUyOhKUiGjK0mFjK4kFTK6klRo5OhGxIUR8b2I+FZbvyIiXoyI+Yh4PCIuauMfaOvzbfvGbqYuSdPnTK50vwi8umx9D3B/Zn4EeAfY2sa3Au+08fvbfpIkRoxuRKwHPgv8ZVsP4HrgG22XfcCtbfmWtk7bfkPbX5JWvTUj7vdnwJeBX2jrlwLDzDzW1g8B69ryOuAgQGYei4iFtv+PT3TgwWBwFtOWzpznmqr0+/2TbjttdCPit4A3M/M7EXHdGOcFnHpy0jh5rul8MMqV7qeB346Im4GLgV8Evgr0ImJNu9pdDxxu+x8GNgCHImINMAO8NfaZS9IUOu093czclZnrM3MjcDvwbGbeATwHfK7ttgl4si0/1dZp25/NzBzrrCVpSp3Lc7o7gC9FxDyL92wfauMPAZe28S8BO89tipK0csQkLkIXFha88lWpXq/HcDic9DS0Cs3MzLzv6S3fkSZJhYyuJBUyupJUyOhKUiGjK0mFjK4kFTK6klTI6EpSIaMrSYWMriQVMrqSVMjoSlIhoytJhYyuJBUyupJUyOhKUiGjK0mFjK4kFTK6klTI6EpSIaMrSYWMriQVMrqSVMjoSlIhoytJhYyuJBUyupJUyOhKUiGjK0mFjK4kFTK6klTI6EpSIaMrSYWMriQVMrqSVMjoSlIhoytJhYyuJBUyupJUyOhKUqHTRjciLo6If42If4+IlyPij9r4FRHxYkTMR8TjEXFRG/9AW59v2zd2+ytI0vQY5Ur3KHB9Zv4qcBVwY0RcA+wB7s/MjwDvAFvb/luBd9r4/W0/SRIjRDcX/aStrm1fCVwPfKON7wNubcu3tHXa9hsiIsY2Y0maYmtG2SkiLgS+A3wE+HPgv4FhZh5ruxwC1rXldcBBgMw8FhELwKXAj0907MFgcNaTl86E55qq9Pv9k24bKbqZ+VPgqojoAd8EfmU8Uzv15KRx8lzT+eCMnl7IzCHwHPApoBcRS9FeDxxuy4eBDQBt+wzw1lhmK0lTbpSnF36pXeESET8HfAZ4lcX4fq7ttgl4si0/1dZp25/NzBznpCVpWo1ye+FyYF+7r3sB8ERmfisiXgH+JiL+BPge8FDb/yHgryJiHngbuL2DeUvSVIpJXIQuLCx45atSvV6P4XA46WloFZqZmXnf01u+I02SChldSSpkdCWpkNGVpEJGV5IKGV1JKmR0JamQ0ZWkQkZXkgoZXUkqZHQlqZDRlaRCRleSChldSSpkdCWpkNGVpEJGV5IKGV1JKmR0JamQ0ZWkQkZXkgoZXUkqZHQlqdCaSU9AWrJx40aGw2Fnx+/1ep0e+7XXXuvs+Fo5jK7OG8PhsLPoDgYD+v1+J8eGboOulcXbC5JUyOhKUiGjK0mFjK4kFTK6klTI6EpSIaMrSYWMriQVMrqSVMjoSlIhoytJhYyuJBUyupJUyOhKUiGjK0mFjK4kFTptdCNiQ0Q8FxGvRMTLEfHFNn5JRDwTEYP2/UNtPCLigYiYj4jvR8THu/4lJGlajHKlewz4/cz8KHANcHdEfBTYCRzIzD5woK0D3AT029ddwINjn7UkTanTRjczj2Tmd9vyu8CrwDrgFmBf220fcGtbvgXYn4teAHoRcfnYZy5JU+iMPiMtIjYCHwNeBGYz80jb9AYw25bXAQeX/dihNnaEExgMBmcyBa1wXZ4PXZ9rnstacqrP4xs5uhHxQeBvgd/LzP+JiJ9ty8yMiBz35LT6dHU+dP3BlOC5rNGM9PRCRKxlMbiPZubfteEfLd02aN/fbOOHgQ3Lfnx9G5OkVW+UpxcCeAh4NTP/dNmmp4BNbXkT8OSy8S+0pxiuARaW3YaQpFVtlNsLnwZ+B/iPiPi3NvYHwD3AExGxFXgduK1texq4GZgH3gM2j3XGkjTFThvdzPxnIE6y+YYT7J/A3ec4L0lakXxHmiQVMrqSVMjoSlIhoytJhYyuJBUyupJUyOhKUiGjK0mFjK4kFTK6klTI6EpSIaMrSYWMriQVMrqSVMjoSlIhoytJhYyuJBUyupJUyOhKUiGjK0mFjK4kFTK6klTI6EpSoTWTnoC05CtXfpgPbrquk2N/rJOj/p+vXPnhjl9BK0VkZvmLLiws1L+oznu9Xo/hcNjJsQeDAf1+v5NjQ7dz13SbmZmJ5eveXpCkQkZXkgoZXUkqZHQlqZDRlaRCRleSChldSSpkdCWpkNGVpEJGV5IKGV1JKmR0JamQ0ZWkQkZXkgoZXUkqdNroRsTDEfFmRPxg2dglEfFMRAza9w+18YiIByJiPiK+HxEf73LykjRtRrnS/Tpw43FjO4EDmdkHDrR1gJuAfvu6C3hwPNOUpJXhtNHNzG8Dbx83fAuwry3vA25dNr4/F70A9CLi8nFNVpKm3dl+RtpsZh5py28As215HXBw2X6H2tgRTmIwGJzlFLQSdXk+dH2ueS5ryak+GuqcP5gyMzMizvozz7r83CpNn67Oh64/Iw08lzWas3164UdLtw3a9zfb+GFgw7L91rcxSRJnH92ngE1teRPw5LLxL7SnGK4BFpbdhpCkVe+0txci4jHgOuCyiDgE/CFwD/BERGwFXgdua7s/DdwMzAPvAZs7mLMkTa3TRjczP3+STTecYN8E7j7XSUnSSuU70iSpkNGVpEJGV5IKGV1JKmR0JamQ0ZWkQkZXkgoZXUkqdM5/8EYap16vN+kpnJVpnbfqGV2dN4bDYWfH7vV6nR5fGpW3FySpkNGVpEJGV5IKGV1JKmR0JamQ0ZWkQkZXkgoZXUkqZHQlqZDRlaRCRleSChldSSpkdCWpkNGVpEJGV5IKGV1JKmR0JamQ0ZWkQkZXkgoZXUkqZHQlqZDRlaRCRleSChldSSpkdCWpkNGVpEJGV5IKGV1JKmR0JamQ0ZWkQkZXkgp1Et2IuDEifhgR8xGxs4vXkKRpFJk53gNGXAj8F/AZ4BDwEvD5zHxlaZ+FhYXxvqhWnd27d7Nnz57Ojr9jxw527drV2fG1eszMzMTy9S6i+ylgLjN/s63vAsjM3Uv7GF1VGwwG9Pv9SU9Dq9Dx0V3TwWusAw4uWz8EfPJkOw8Ggw6mIP1/nmuqcqp/4LuI7hnx6kMVvNLV+aKL/0g7DGxYtr6+jUnSqtdFdF8C+hFxRURcBNwOPNXB60jS1Bn77YXMPBYR24B/AC4EHs7Ml8f9OpI0jTq5p5uZTwNPd3FsSZpmviNNkgoZXUkqZHQlqdDY35E2Ct+RJmm1OP4daV7pSlIhoytJhSZye0GSViuvdCWpkNHVihYRD0fEmxHxg0nPRQKjq5Xv68CNk56EtMToakXLzG8Db096HtISoytJhYyuJBUyupJUyOhKUiGjqxUtIh4D/gW4MiIORcTWSc9Jq5vvSJOkQl7pSlIhoytJhYyuJBUyupJUyOhKUiGjK0mFjK4kFTK6klTofwFQ+fUNPYELhgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 360x360 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_QecEVtnBXi_",
        "outputId": "ced1318a-6336-4a9a-db09-62fba83559ac"
      },
      "source": [
        "#it is worth noting that twitter platform allows only 280 charecters to be present in the tweet, however in dataset we had imported we find close to 12% of entries that contain more than 280 charecters. Hence it is required we perform data wrangling to remove unwanted information.\n",
        "x= df[df.pre_clean_len > 280]\n",
        "len(x)/len(df)"
      ],
      "execution_count": 144,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.11970610660566992"
            ]
          },
          "metadata": {},
          "execution_count": 144
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "7lFBkQzuBa0Y",
        "outputId": "ceb83ae7-a239-428d-aec0-3a6cb59148c8"
      },
      "source": [
        "#sorting dataset by timestamp metric 'date'\n",
        "df.sort_values(by=['date'], inplace=True, ascending=True)\n",
        "df.reset_index(drop=True, inplace=True)\n",
        "df.head()"
      ],
      "execution_count": 145,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>username</th>\n",
              "      <th>tweet</th>\n",
              "      <th>replies_count</th>\n",
              "      <th>retweets_count</th>\n",
              "      <th>likes_count</th>\n",
              "      <th>pre_clean_len</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2020-01-01 08:32:00</td>\n",
              "      <td>nebraskangooner</td>\n",
              "      <td>@PhilipEytan Almost as useless as your tweet a...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>122</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2020-01-01 11:30:00</td>\n",
              "      <td>petermccormack</td>\n",
              "      <td>@walshjonwalsh In the reports, I account for i...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>179</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2020-01-01 14:51:00</td>\n",
              "      <td>100trillionusd</td>\n",
              "      <td>@BrendanLane0 Good eye. Yes I used the origina...</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "      <td>88</td>\n",
              "      <td>293</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2020-01-01 15:29:00</td>\n",
              "      <td>coryklippsten</td>\n",
              "      <td>@gauravdahake @bitdroplet @cryptovill_es @pier...</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>298</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2020-01-01 22:09:00</td>\n",
              "      <td>woonomic</td>\n",
              "      <td>@henriqueztony @NexoFinance No, because it has...</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>10</td>\n",
              "      <td>260</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 date         username  ... likes_count  pre_clean_len\n",
              "0 2020-01-01 08:32:00  nebraskangooner  ...           2            122\n",
              "1 2020-01-01 11:30:00   petermccormack  ...           3            179\n",
              "2 2020-01-01 14:51:00   100trillionusd  ...          88            293\n",
              "3 2020-01-01 15:29:00    coryklippsten  ...           1            298\n",
              "4 2020-01-01 22:09:00         woonomic  ...          10            260\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 145
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_IbXGeA2BdF7"
      },
      "source": [
        "#data wrangling function to remove unwanted information from acquired tweets \n",
        "plt.style.use('fivethirtyeight')\n",
        "%matplotlib inline\n",
        "#importing required libraries \n",
        "import re\n",
        "from bs4 import BeautifulSoup\n",
        "from nltk.tokenize import WordPunctTokenizer\n",
        "#initilising tokenizer\n",
        "tok = WordPunctTokenizer()\n",
        "\n",
        "pat1 = r'@[A-Za-z0-9_]+'\n",
        "pat2 = r'https?://[^ ]+'\n",
        "combined_pat = r'|'.join((pat1, pat2))\n",
        "www_pat = r'www.[^ ]+'\n",
        "negations_dic = {\"isn't\":\"is not\", \"aren't\":\"are not\", \"wasn't\":\"was not\", \"weren't\":\"were not\",\n",
        "                \"haven't\":\"have not\",\"hasn't\":\"has not\",\"hadn't\":\"had not\",\"won't\":\"will not\",\n",
        "                \"wouldn't\":\"would not\", \"don't\":\"do not\", \"doesn't\":\"does not\",\"didn't\":\"did not\",\n",
        "                \"can't\":\"can not\",\"couldn't\":\"could not\",\"shouldn't\":\"should not\",\"mightn't\":\"might not\",\n",
        "                \"mustn't\":\"must not\"}\n",
        "neg_pattern = re.compile(r'\\b(' + '|'.join(negations_dic.keys()) + r')\\b')\n",
        "\n",
        "#tasks performed by function listed as follows:\n",
        "#remove HTML decoing \n",
        "#remove @mentions\n",
        "#remove URL links specified in tweets \n",
        "#remove UTF-8 BOM(Byte Order Mark)\n",
        "#remove hastags and numbers specified in tweets\n",
        "#remove numbers and special charecters\n",
        "#convert upper case to lower case charecters \n",
        "#we have defined dictionary above that contain negation words, so that remove of special charecters from tweets - we do not end up loosing text value\n",
        "#tokenising text post cleaning tweet data and joing them to preserve value  \n",
        "\n",
        "def tweet_cleaner(text):\n",
        "    soup = BeautifulSoup(text, 'lxml')\n",
        "    souped = soup.get_text()\n",
        "    try:\n",
        "        bom_removed = souped.decode(\"utf-8-sig\").replace(u\"\\ufffd\", \"?\")\n",
        "    except:\n",
        "        bom_removed = souped\n",
        "    stripped = re.sub(combined_pat, '', bom_removed)\n",
        "    stripped = re.sub(www_pat, '', stripped)\n",
        "    lower_case = stripped.lower()\n",
        "    neg_handled = neg_pattern.sub(lambda x: negations_dic[x.group()], lower_case)\n",
        "    letters_only = re.sub(\"[^a-zA-Z]\", \" \", neg_handled)\n",
        "    words = [x for x  in tok.tokenize(letters_only) if len(x) > 1]\n",
        "    return (\" \".join(words)).strip()"
      ],
      "execution_count": 146,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "797sr3McBsAK",
        "outputId": "428df2ba-8c42-4320-e3dc-8181b26618c5"
      },
      "source": [
        "#applying data wrangling fucntion tweet cleaner over tweet field in dataset \n",
        "df['tweet'] = df['tweet'].apply(lambda text: tweet_cleaner(text))\n",
        "df.head()"
      ],
      "execution_count": 147,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>username</th>\n",
              "      <th>tweet</th>\n",
              "      <th>replies_count</th>\n",
              "      <th>retweets_count</th>\n",
              "      <th>likes_count</th>\n",
              "      <th>pre_clean_len</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2020-01-01 08:32:00</td>\n",
              "      <td>nebraskangooner</td>\n",
              "      <td>almost as useless as your tweet about btc bein...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>122</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2020-01-01 11:30:00</td>\n",
              "      <td>petermccormack</td>\n",
              "      <td>in the reports account for it as the value of ...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>179</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2020-01-01 14:51:00</td>\n",
              "      <td>100trillionusd</td>\n",
              "      <td>good eye yes used the original formula btc sf ...</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "      <td>88</td>\n",
              "      <td>293</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2020-01-01 15:29:00</td>\n",
              "      <td>coryklippsten</td>\n",
              "      <td>that great congrats gaurav are you able to set...</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>298</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2020-01-01 22:09:00</td>\n",
              "      <td>woonomic</td>\n",
              "      <td>no because it has not established oscillation ...</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>10</td>\n",
              "      <td>260</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 date         username  ... likes_count  pre_clean_len\n",
              "0 2020-01-01 08:32:00  nebraskangooner  ...           2            122\n",
              "1 2020-01-01 11:30:00   petermccormack  ...           3            179\n",
              "2 2020-01-01 14:51:00   100trillionusd  ...          88            293\n",
              "3 2020-01-01 15:29:00    coryklippsten  ...           1            298\n",
              "4 2020-01-01 22:09:00         woonomic  ...          10            260\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 147
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "NF4qtb-TBuol",
        "outputId": "c1c77b92-0297-433d-c2eb-0e8660ca0ec4"
      },
      "source": [
        "#creating a column named post_clean_len describing length of tweets post data wrangling\n",
        "df['post_clean_len'] = [len(t) for t in df.tweet]\n",
        "df.head()"
      ],
      "execution_count": 148,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>username</th>\n",
              "      <th>tweet</th>\n",
              "      <th>replies_count</th>\n",
              "      <th>retweets_count</th>\n",
              "      <th>likes_count</th>\n",
              "      <th>pre_clean_len</th>\n",
              "      <th>post_clean_len</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2020-01-01 08:32:00</td>\n",
              "      <td>nebraskangooner</td>\n",
              "      <td>almost as useless as your tweet about btc bein...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>122</td>\n",
              "      <td>73</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2020-01-01 11:30:00</td>\n",
              "      <td>petermccormack</td>\n",
              "      <td>in the reports account for it as the value of ...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>179</td>\n",
              "      <td>156</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2020-01-01 14:51:00</td>\n",
              "      <td>100trillionusd</td>\n",
              "      <td>good eye yes used the original formula btc sf ...</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "      <td>88</td>\n",
              "      <td>293</td>\n",
              "      <td>233</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2020-01-01 15:29:00</td>\n",
              "      <td>coryklippsten</td>\n",
              "      <td>that great congrats gaurav are you able to set...</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>298</td>\n",
              "      <td>134</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2020-01-01 22:09:00</td>\n",
              "      <td>woonomic</td>\n",
              "      <td>no because it has not established oscillation ...</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>10</td>\n",
              "      <td>260</td>\n",
              "      <td>222</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 date         username  ... pre_clean_len  post_clean_len\n",
              "0 2020-01-01 08:32:00  nebraskangooner  ...           122              73\n",
              "1 2020-01-01 11:30:00   petermccormack  ...           179             156\n",
              "2 2020-01-01 14:51:00   100trillionusd  ...           293             233\n",
              "3 2020-01-01 15:29:00    coryklippsten  ...           298             134\n",
              "4 2020-01-01 22:09:00         woonomic  ...           260             222\n",
              "\n",
              "[5 rows x 8 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 148
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 316
        },
        "id": "IayXrfRuBxro",
        "outputId": "a77b2c04-1dab-44ff-c0ba-15396bea602f"
      },
      "source": [
        "# plotting distribution of length of tweets post data wrangling\n",
        "fig, ax = plt.subplots(figsize=(5, 5))\n",
        "plt.boxplot(df.post_clean_len)\n",
        "plt.show()"
      ],
      "execution_count": 149,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV4AAAErCAYAAACWxOtoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAMiElEQVR4nO3dX4ild33H8c+3xnpRZY4iXdNkIV5MhXjRKBIFe5FSqDE3q1BCvNBFU+OFAQUvTLxQQUSFqiC0oQkGI1htQMVQQts0CN7UP20Imj+Vs+hKdlmzUJ1RECxJf73YZ+MhxJ3JzDzfycy8XjCc5/ye5zzPb+HZN4ffnLNbY4wA0OcP9nsCAEeN8AI0E16AZsIL0Oyy/bjo5ubmM7/RW1tbq/2YA8BctmrcvoR31eoEAY4CSw0AzYQXoJnwciQsl8v9ngI8Q3gBmgkvQDPhBWgmvADNhBegmfACNBNegGbCC9BMeAGa7fs/kgM79alPfSqf+cxnZjn3hz/84dx+++2znBtqP/7PNf8iGd0Wi0U2Njb2exocQc/1z0JaagBoJrwAzYQXoJnwAjQTXoBmwgvQTHgBmgkvQDPhBWgmvADNhBegmfACNBNegGbCC9BMeAGaCS9AM+EFaCa8AM22DG9VHa+qb1fVY1X1aFV9YBr/eFWdraqHp58bVl5ze1WdqqofV9Vb5vwDABw02/nPLp9K8qExxkNV9bIk/1VVD0z7Pj/G+NvVg6vq6iQ3JXltkj9J8u9V9adjjKf3cuIAB9WW73jHGOfGGA9N279O8niSKy7xkhNJvjbG+O0Y46dJTiW5di8mC3AYPK//3r2qrkryuiTfS/LmJLdW1buS/GcuvCv+ZS5E+bsrLzuTS4R6uVw+vxnDDrnX6LK+vn7J/dsOb1W9NMnXk3xwjPGrqrojySeSjOnxs0nes9cThL3iXuOFYlufaqiqF+dCdL8yxvhGkowxnhxjPD3G+L8kd+V3ywlnkxxfefmV0xgA2d6nGirJF5M8Psb43Mr45SuHvT3JI9P2fUluqqqXVNWrk6wn+f7eTRngYNvOUsObk7wzyY+q6uFp7CNJ3lFV1+TCUsPpJO9LkjHGo1V1b5LHcuETEe/3iQaA36kxRvtFNzc3+y/KkbZYLLKxsbHf0+AIWltbq2eP+eYaQDPhBWgmvADNhBegmfACNBNegGbCC9BMeAGaCS9AM+EFaCa8AM2EF6CZ8AI0E16AZsIL0Ex4AZoJL0Az4QVoJrwAzYQXoJnwAjQTXoBmwgvQTHgBmgkvQDPhBWgmvADNhBegmfACNBNegGbCC9BMeAGaCS9AM+EFaCa8AM2EF6DZluGtquNV9e2qeqyqHq2qD0zjr6iqB6pqOT2+fBqvqvpCVZ2qqh9W1evn/kMAHCTbecf7VJIPjTGuTvKmJO+vqquT3JbkwTHGepIHp+dJ8tYk69PPLUnu2PNZAxxgW4Z3jHFujPHQtP3rJI8nuSLJiST3TIfdk+Rt0/aJJF8eF3w3yaKqLt/zmQMcUJc9n4Or6qokr0vyvSTHxhjnpl0/T3Js2r4iyRMrLzszjZ3Lc1gul89nCrBj7jW6rK+vX3L/tsNbVS9N8vUkHxxj/Kqqntk3xhhVNeaYIOwV9xovFNv6VENVvTgXovuVMcY3puEnLy4hTI/np/GzSY6vvPzKaQyAbO9TDZXki0keH2N8bmXXfUlOTtsnk3xrZfxd06cb3pRkc2VJAuDI285Sw5uTvDPJj6rq4WnsI0k+neTeqro5yc+S3Djtuz/JDUlOJflNknfv6YwBDrgaY0dLs7uyubnZf1GOtMVikY2Njf2eBkfQ2tpaPXvMN9cAmgkvQDPhBWgmvADNntc312BOV1111ay/AFssFrOe+/Tp07Odn8NFeHnB2NjYmC28y+Vy1m+uzRl1Dh9LDQDNhBegmfACNBNegGbCC9BMeAGaCS9AM+EFaCa8AM2EF6CZ8AI0E16AZsIL0Ex4AZoJL0Az4QVoJrwAzYQXoJnwAjQTXoBmwgvQTHgBmgkvQDPhBWgmvADNhBegmfACNBNegGbCC9Bsy/BW1d1Vdb6qHlkZ+3hVna2qh6efG1b23V5Vp6rqx1X1lrkmDnBQbecd75eSXP8c458fY1wz/dyfJFV1dZKbkrx2es3fV9WL9mqyAIfBluEdY3wnyS+2eb4TSb42xvjtGOOnSU4luXYX8wM4dHazxntrVf1wWop4+TR2RZInVo45M40BMLlsh6+7I8knkozp8bNJ3rOTEy2Xyx1OgcNozvth7nvNvcxF6+vrl9y/o/COMZ68uF1VdyX55+np2STHVw69chrb8QQ5Wua6H5bL5ez3mnuZ7drRUkNVXb7y9O1JLn7i4b4kN1XVS6rq1UnWk3x/d1MEOFy2fMdbVV9Ncl2SV1bVmSQfS3JdVV2TC0sNp5O8L0nGGI9W1b1JHkvyVJL3jzGenmfqAAfTluEdY7zjOYa/eInjP5nkk7uZFMBh5ptrAM2EF6CZ8AI0E16AZsIL0Ex4AZoJL0Az4QVoJrwAzYQXoJnwAjQTXoBmwgvQTHgBmgkvQDPhBWgmvADNhBegmfACNBNegGbCC9BMeAGaCS9AM+EFaCa8AM2EF6CZ8AI0E16AZsIL0Ex4AZoJL0Az4QVoJrwAzYQXoJnwAjQTXoBmW4a3qu6uqvNV9cjK2Cuq6oGqWk6PL5/Gq6q+UFWnquqHVfX6OScPcBBt5x3vl5Jc/6yx25I8OMZYT/Lg9DxJ3ppkffq5JckdezNNgMNjy/COMb6T5BfPGj6R5J5p+54kb1sZ//K44LtJFlV1+V5NFuAwuGyHrzs2xjg3bf88ybFp+4okT6wcd2YaO5ffY7lc7nAKHEZz3g9z32vuZS5aX1+/5P6dhvcZY4xRVWOnr99qghwtc90Py+Vy9nvNvcx27fRTDU9eXEKYHs9P42eTHF857sppDIDJTsN7X5KT0/bJJN9aGX/X9OmGNyXZXFmSACDbWGqoqq8muS7JK6vqTJKPJfl0knur6uYkP0ty43T4/UluSHIqyW+SvHuGOXNIffQ1r8pLT143y7lfN8tZf+ejr3nVzFfgMKkxdrw8u2Obm5v9F+UFb7FYZGNjY5Zzz73GO+fcOdjW1tbq2WO+uQbQTHgBmgkvQDPhBWgmvADNhBegmfACNBNegGbCC9BMeAGaCS9AM+EFaCa8AM2EF6CZ8AI0E16AZsIL0Ex4AZoJL0Az4QVoJrwAzYQXoJnwAjQTXoBmwgvQTHgBmgkvQDPhBWgmvADNhBegmfACNBNegGbCC9BMeAGaCS9AM+EFaHbZbl5cVaeT/DrJ00meGmO8oapekeSfklyV5HSSG8cYv9zdNAEOj714x/sXY4xrxhhvmJ7fluTBMcZ6kgen5wBM5lhqOJHknmn7niRvm+EaAAfWrpYakowk/1ZVI8k/jDHuTHJsjHFu2v/zJMcudYLlcrnLKXCYzHk/zH2vuZe5aH19/ZL7dxvePx9jnK2qP07yQFX99+rOMcaYorzjCXK0zHU/LJfL2e819zLbtaulhjHG2enxfJJvJrk2yZNVdXmSTI/ndztJgMNkx+Gtqj+qqpdd3E7yV0keSXJfkpPTYSeTfGu3kwQ4THaz1HAsyTer6uJ5/nGM8S9V9YMk91bVzUl+luTG3U8T4PDYcXjHGD9J8mfPMf4/Sf5yN5MCOMx8cw2gmfACNNvtx8lgTy0Wi/2ewo4c1HmzP4SXF4yNjY3Zzr1YLGY9PzwflhoAmgkvQDPhBWgmvADNhBegmfACNBNegGbCC9BMeAGaCS9AM+EFaCa8AM2EF6CZ8AI0E16AZsIL0Ex4AZoJL0Az4QVoJrwAzYQXoJnwAjQTXoBmwgvQTHgBmgkvQDPhBWgmvADNhBegmfACNBNegGbCC9BstvBW1fVV9eOqOlVVt811HYCDZpbwVtWLkvxdkrcmuTrJO6rq6jmuBXDQzPWO99okp8YYPxlj/G+SryU5MdO1AA6Uy2Y67xVJnlh5fibJG5/rwOVyOdMUOOzuvPPO3HXXXds+frFYbPvY9773vbnlllt2Mi3I+vr6JffXGGPPL1pVf53k+jHG30zP35nkjWOMW5Nkc3Nz7y8Kl7BcLrf8ywBzWFtbq2ePzbXUcDbJ8ZXnV05jAEfeXOH9QZL1qnp1Vf1hkpuS3DfTtQAOlFnWeMcYT1XVrUn+NcmLktw9xnh0jmsBHDRz/XItY4z7k9w/1/kBDirfXANoJrwAzYQXoJnwAjQTXoBms3xzbSu+uQYcFZ3fXAPg9xBegGb7stQAcJR5xwvQTHg51Krq7qo6X1WP7Pdc4CLh5bD7UpLr93sSsEp4OdTGGN9J8ov9ngesEl6AZsIL0Ex4AZoJL0Az4eVQq6qvJvmPJK+pqjNVdfN+zwl8cw2gmXe8AM2EF6CZ8AI0E16AZsIL0Ex4AZoJL0Az4QVo9v+N3ZlKk0dL3gAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 360x360 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W-9WolZMBzxh",
        "outputId": "f1188ebb-39e8-4b96-e96e-5f60036dc7f8"
      },
      "source": [
        "#dropping columns from dataset that are not required for further analysis \n",
        "df.drop(columns=['username', 'pre_clean_len','post_clean_len'], inplace= True)\n",
        "df.shape"
      ],
      "execution_count": 150,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(28854, 5)"
            ]
          },
          "metadata": {},
          "execution_count": 150
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "HSoHCc0bB2KQ",
        "outputId": "e10b3c3f-9aba-4d69-d0c8-261c4b7a82e2"
      },
      "source": [
        "#aggregating fields by timestamp 'date'\n",
        "df = df.groupby(['date'], as_index = False).agg({'tweet': ''.join, 'replies_count': \"sum\", 'retweets_count': \"sum\",'likes_count':\"sum\"}, Inplace=True)\n",
        "df.head()"
      ],
      "execution_count": 151,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>tweet</th>\n",
              "      <th>replies_count</th>\n",
              "      <th>retweets_count</th>\n",
              "      <th>likes_count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2020-01-01 08:32:00</td>\n",
              "      <td>almost as useless as your tweet about btc bein...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2020-01-01 11:30:00</td>\n",
              "      <td>in the reports account for it as the value of ...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2020-01-01 14:51:00</td>\n",
              "      <td>good eye yes used the original formula btc sf ...</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "      <td>88</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2020-01-01 15:29:00</td>\n",
              "      <td>that great congrats gaurav are you able to set...</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2020-01-01 22:09:00</td>\n",
              "      <td>no because it has not established oscillation ...</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 date  ... likes_count\n",
              "0 2020-01-01 08:32:00  ...           2\n",
              "1 2020-01-01 11:30:00  ...           3\n",
              "2 2020-01-01 14:51:00  ...          88\n",
              "3 2020-01-01 15:29:00  ...           1\n",
              "4 2020-01-01 22:09:00  ...          10\n",
              "\n",
              "[5 rows x 5 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 151
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2fXHq6NuB4zk",
        "outputId": "b92dba83-da78-41d6-8826-cf48ce87b3cc"
      },
      "source": [
        "#computing sentiment metrics of tweet usign vader library available as python package\n",
        "df['Comp'] = ''\n",
        "df['Negative'] = ''\n",
        "df['Neutral'] = ''\n",
        "df['Positive'] = ''\n",
        "\n",
        "import nltk\n",
        "nltk.download('vader_lexicon')\n",
        "\n",
        "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
        "import unicodedata\n",
        "sentiment_i_a = SentimentIntensityAnalyzer()\n",
        "for indexx, row in df.T.iteritems():\n",
        "    try:\n",
        "        sentence_i = unicodedata.normalize('NFKD', df.loc[indexx, 'tweet'])\n",
        "        sentence_sentiment = sentiment_i_a.polarity_scores(sentence_i)\n",
        "        df.at[indexx, 'Comp'] = sentence_sentiment['compound']\n",
        "        df.at[indexx, 'Negative'] = sentence_sentiment['neg']\n",
        "        df.at[indexx, 'Neutral'] = sentence_sentiment['neu']\n",
        "        df.at[indexx, 'Positive'] = sentence_sentiment['pos']\n",
        "    except TypeError:\n",
        "        print (df.loc[indexx, 'tweet'])\n",
        "        print (indexx)"
      ],
      "execution_count": 152,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package vader_lexicon to /root/nltk_data...\n",
            "[nltk_data]   Package vader_lexicon is already up-to-date!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "qII-Upv2B7fd",
        "outputId": "93a6315e-c32b-469a-ff23-2ac09d2ce837"
      },
      "source": [
        "#visualising dataset post creation of sentiment metrics\n",
        "df.head()"
      ],
      "execution_count": 153,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>tweet</th>\n",
              "      <th>replies_count</th>\n",
              "      <th>retweets_count</th>\n",
              "      <th>likes_count</th>\n",
              "      <th>Comp</th>\n",
              "      <th>Negative</th>\n",
              "      <th>Neutral</th>\n",
              "      <th>Positive</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2020-01-01 08:32:00</td>\n",
              "      <td>almost as useless as your tweet about btc bein...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>0.3774</td>\n",
              "      <td>0.135</td>\n",
              "      <td>0.644</td>\n",
              "      <td>0.22</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2020-01-01 11:30:00</td>\n",
              "      <td>in the reports account for it as the value of ...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>0.4588</td>\n",
              "      <td>0.065</td>\n",
              "      <td>0.787</td>\n",
              "      <td>0.149</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2020-01-01 14:51:00</td>\n",
              "      <td>good eye yes used the original formula btc sf ...</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "      <td>88</td>\n",
              "      <td>0.802</td>\n",
              "      <td>0.079</td>\n",
              "      <td>0.669</td>\n",
              "      <td>0.251</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2020-01-01 15:29:00</td>\n",
              "      <td>that great congrats gaurav are you able to set...</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0.8176</td>\n",
              "      <td>0</td>\n",
              "      <td>0.717</td>\n",
              "      <td>0.283</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2020-01-01 22:09:00</td>\n",
              "      <td>no because it has not established oscillation ...</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>10</td>\n",
              "      <td>-0.296</td>\n",
              "      <td>0.049</td>\n",
              "      <td>0.951</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 date  ... Positive\n",
              "0 2020-01-01 08:32:00  ...     0.22\n",
              "1 2020-01-01 11:30:00  ...    0.149\n",
              "2 2020-01-01 14:51:00  ...    0.251\n",
              "3 2020-01-01 15:29:00  ...    0.283\n",
              "4 2020-01-01 22:09:00  ...        0\n",
              "\n",
              "[5 rows x 9 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 153
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "dyGkn4kYB_R2",
        "outputId": "627f1511-fe38-4719-9088-3ba4789d9739"
      },
      "source": [
        "#adding constant value of 1 to social media metrics namely, 'replies_count', 'retweets_count', 'likes_count' for smoothening & computation of text feature  \n",
        "df['replies_count'] = df['replies_count'] + 1 \n",
        "df['retweets_count'] = df['retweets_count'] + 1 \n",
        "df['likes_count'] = df['likes_count'] + 1 \n",
        "df['Comp'] = df['Comp']+1\n",
        "df.head()"
      ],
      "execution_count": 154,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>tweet</th>\n",
              "      <th>replies_count</th>\n",
              "      <th>retweets_count</th>\n",
              "      <th>likes_count</th>\n",
              "      <th>Comp</th>\n",
              "      <th>Negative</th>\n",
              "      <th>Neutral</th>\n",
              "      <th>Positive</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2020-01-01 08:32:00</td>\n",
              "      <td>almost as useless as your tweet about btc bein...</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>1.3774</td>\n",
              "      <td>0.135</td>\n",
              "      <td>0.644</td>\n",
              "      <td>0.22</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2020-01-01 11:30:00</td>\n",
              "      <td>in the reports account for it as the value of ...</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>1.4588</td>\n",
              "      <td>0.065</td>\n",
              "      <td>0.787</td>\n",
              "      <td>0.149</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2020-01-01 14:51:00</td>\n",
              "      <td>good eye yes used the original formula btc sf ...</td>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>89</td>\n",
              "      <td>1.802</td>\n",
              "      <td>0.079</td>\n",
              "      <td>0.669</td>\n",
              "      <td>0.251</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2020-01-01 15:29:00</td>\n",
              "      <td>that great congrats gaurav are you able to set...</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1.8176</td>\n",
              "      <td>0</td>\n",
              "      <td>0.717</td>\n",
              "      <td>0.283</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2020-01-01 22:09:00</td>\n",
              "      <td>no because it has not established oscillation ...</td>\n",
              "      <td>4</td>\n",
              "      <td>3</td>\n",
              "      <td>11</td>\n",
              "      <td>0.704</td>\n",
              "      <td>0.049</td>\n",
              "      <td>0.951</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 date  ... Positive\n",
              "0 2020-01-01 08:32:00  ...     0.22\n",
              "1 2020-01-01 11:30:00  ...    0.149\n",
              "2 2020-01-01 14:51:00  ...    0.251\n",
              "3 2020-01-01 15:29:00  ...    0.283\n",
              "4 2020-01-01 22:09:00  ...        0\n",
              "\n",
              "[5 rows x 9 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 154
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "gNg3LhA6CBcs",
        "outputId": "b4387b58-72e9-4e8f-85e1-491f5baf9bdb"
      },
      "source": [
        "#we compute text feature required for further analysis, as a product of composite value derived from snetiment analysis alogside social media metrics such as 'replies_count', 'retweets_count', 'likes_count' \n",
        "df['text_feature']=df['Comp']*df['replies_count']*df['retweets_count']*df['likes_count']\n",
        "df.head()"
      ],
      "execution_count": 155,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>tweet</th>\n",
              "      <th>replies_count</th>\n",
              "      <th>retweets_count</th>\n",
              "      <th>likes_count</th>\n",
              "      <th>Comp</th>\n",
              "      <th>Negative</th>\n",
              "      <th>Neutral</th>\n",
              "      <th>Positive</th>\n",
              "      <th>text_feature</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2020-01-01 08:32:00</td>\n",
              "      <td>almost as useless as your tweet about btc bein...</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>1.3774</td>\n",
              "      <td>0.135</td>\n",
              "      <td>0.644</td>\n",
              "      <td>0.22</td>\n",
              "      <td>8.2644</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2020-01-01 11:30:00</td>\n",
              "      <td>in the reports account for it as the value of ...</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>1.4588</td>\n",
              "      <td>0.065</td>\n",
              "      <td>0.787</td>\n",
              "      <td>0.149</td>\n",
              "      <td>11.6704</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2020-01-01 14:51:00</td>\n",
              "      <td>good eye yes used the original formula btc sf ...</td>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>89</td>\n",
              "      <td>1.802</td>\n",
              "      <td>0.079</td>\n",
              "      <td>0.669</td>\n",
              "      <td>0.251</td>\n",
              "      <td>5773.61</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2020-01-01 15:29:00</td>\n",
              "      <td>that great congrats gaurav are you able to set...</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1.8176</td>\n",
              "      <td>0</td>\n",
              "      <td>0.717</td>\n",
              "      <td>0.283</td>\n",
              "      <td>10.9056</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2020-01-01 22:09:00</td>\n",
              "      <td>no because it has not established oscillation ...</td>\n",
              "      <td>4</td>\n",
              "      <td>3</td>\n",
              "      <td>11</td>\n",
              "      <td>0.704</td>\n",
              "      <td>0.049</td>\n",
              "      <td>0.951</td>\n",
              "      <td>0</td>\n",
              "      <td>92.928</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 date  ... text_feature\n",
              "0 2020-01-01 08:32:00  ...       8.2644\n",
              "1 2020-01-01 11:30:00  ...      11.6704\n",
              "2 2020-01-01 14:51:00  ...      5773.61\n",
              "3 2020-01-01 15:29:00  ...      10.9056\n",
              "4 2020-01-01 22:09:00  ...       92.928\n",
              "\n",
              "[5 rows x 10 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 155
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "he0KNEeZCDV1",
        "outputId": "997afaaf-d148-440e-d7e9-3c143ab90a2f"
      },
      "source": [
        "#filtering fields that are required for further analysis \n",
        "df1= df.filter(items=['date', 'text_feature'])\n",
        "df1.head()"
      ],
      "execution_count": 156,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>text_feature</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2020-01-01 08:32:00</td>\n",
              "      <td>8.2644</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2020-01-01 11:30:00</td>\n",
              "      <td>11.6704</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2020-01-01 14:51:00</td>\n",
              "      <td>5773.61</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2020-01-01 15:29:00</td>\n",
              "      <td>10.9056</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2020-01-01 22:09:00</td>\n",
              "      <td>92.928</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 date text_feature\n",
              "0 2020-01-01 08:32:00       8.2644\n",
              "1 2020-01-01 11:30:00      11.6704\n",
              "2 2020-01-01 14:51:00      5773.61\n",
              "3 2020-01-01 15:29:00      10.9056\n",
              "4 2020-01-01 22:09:00       92.928"
            ]
          },
          "metadata": {},
          "execution_count": 156
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uzLMQjrbCF0U",
        "outputId": "1afb8090-06b2-47ec-ab5c-12905319ae9a"
      },
      "source": [
        "#reading file containing candlesticks data pulled from binance exchange between time period 15-08-2019 to 15-08-2021 at a minute level for Bitcoin(BTC)\n",
        "df2= pd.read_csv('Binance_data_BTCUSDT_15 Aug, 2019_to_15 Aug, 2021_1m.csv')\n",
        "#exploring dimensions of imported dataset\n",
        "df2.shape"
      ],
      "execution_count": 157,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1049773, 13)"
            ]
          },
          "metadata": {},
          "execution_count": 157
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "2yh2xyn7EwzG",
        "outputId": "842d66cd-c098-4360-ef46-34b078a76038"
      },
      "source": [
        "#visualising dataset imported\n",
        "df1.head()"
      ],
      "execution_count": 158,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>text_feature</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2020-01-01 08:32:00</td>\n",
              "      <td>8.2644</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2020-01-01 11:30:00</td>\n",
              "      <td>11.6704</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2020-01-01 14:51:00</td>\n",
              "      <td>5773.61</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2020-01-01 15:29:00</td>\n",
              "      <td>10.9056</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2020-01-01 22:09:00</td>\n",
              "      <td>92.928</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 date text_feature\n",
              "0 2020-01-01 08:32:00       8.2644\n",
              "1 2020-01-01 11:30:00      11.6704\n",
              "2 2020-01-01 14:51:00      5773.61\n",
              "3 2020-01-01 15:29:00      10.9056\n",
              "4 2020-01-01 22:09:00       92.928"
            ]
          },
          "metadata": {},
          "execution_count": 158
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GsrRvhSxHSv1"
      },
      "source": [
        "#parametres returned by api response from binance exchange\n",
        "#0-   Open Time \n",
        "#1 -  Open Price\n",
        "#2 -  High Price\n",
        "#3 -  Low Price\n",
        "#4 -  Close Price\n",
        "#5 -  Volume of trades\n",
        "#6 -  Close Time\n",
        "#7 -  Quote asset volume \n",
        "#8 -  Number of trades \n",
        "#9 -  Taker Buy Base Asset Volume \n",
        "#10 - Taker Buy Quote Asset Volume \n",
        "#11 - Ignore"
      ],
      "execution_count": 159,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vmFB__JdHVUW",
        "outputId": "4a121f17-9b21-422e-fca0-994a184489c5"
      },
      "source": [
        "#we rename columns with names of the resective parameters \n",
        "df2.rename(columns={'0': 'Open_time', '1':'Open', '2':'High', '3':'Low', '4':'Close_BTC', '5':'Volume_BTC','6':'Close_time','7':'Quote asset volume','8':'Number_of_trades_BTC','9':'Taker buy base asset volume','10':'Taker buy quote asset volume','11': 'Ignore'}, inplace=True)\n",
        "#we drop columns which are not required for further analysis, namely Open Price, High Price, Low Price, Quote Asset Volume, Taker Buy Base Asset Volume, Taker Buy Quote Asset Volume, Ignore \n",
        "df2.drop(columns=['Unnamed: 0','Open','High','Low','Close_time','Quote asset volume','Taker buy base asset volume','Taker buy quote asset volume','Ignore'], inplace=True)\n",
        "#exploring dimensions of dataframe post dropping columns \n",
        "df2.shape"
      ],
      "execution_count": 160,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1049773, 4)"
            ]
          },
          "metadata": {},
          "execution_count": 160
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "uH77Ox38HXqc",
        "outputId": "8ca2bdf4-5137-4806-86c6-6606e50beee5"
      },
      "source": [
        "#converting unixtime to readable time format\n",
        "df2['Open_time'] = pd.to_datetime(df2['Open_time']/1000, unit='s')\n",
        "#visualising dataset \n",
        "df2.head()\n"
      ],
      "execution_count": 161,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Open_time</th>\n",
              "      <th>Close_BTC</th>\n",
              "      <th>Volume_BTC</th>\n",
              "      <th>Number_of_trades_BTC</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2019-08-15 00:00:00</td>\n",
              "      <td>10144.54</td>\n",
              "      <td>114.720030</td>\n",
              "      <td>1459</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2019-08-15 00:01:00</td>\n",
              "      <td>10132.25</td>\n",
              "      <td>409.414904</td>\n",
              "      <td>3443</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2019-08-15 00:02:00</td>\n",
              "      <td>10132.19</td>\n",
              "      <td>118.248972</td>\n",
              "      <td>1028</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2019-08-15 00:03:00</td>\n",
              "      <td>10149.31</td>\n",
              "      <td>36.480215</td>\n",
              "      <td>422</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2019-08-15 00:04:00</td>\n",
              "      <td>10128.26</td>\n",
              "      <td>84.651933</td>\n",
              "      <td>747</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            Open_time  Close_BTC  Volume_BTC  Number_of_trades_BTC\n",
              "0 2019-08-15 00:00:00   10144.54  114.720030                  1459\n",
              "1 2019-08-15 00:01:00   10132.25  409.414904                  3443\n",
              "2 2019-08-15 00:02:00   10132.19  118.248972                  1028\n",
              "3 2019-08-15 00:03:00   10149.31   36.480215                   422\n",
              "4 2019-08-15 00:04:00   10128.26   84.651933                   747"
            ]
          },
          "metadata": {},
          "execution_count": 161
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "F6c1mUy0HjUx",
        "outputId": "a4e94a93-b4b1-4627-fe87-69af3a6a8642"
      },
      "source": [
        "#Renaming column Open_time to date for ease of use \n",
        "df2.rename(columns={'Open_time': 'date'}, inplace= True)\n",
        "df2.head()"
      ],
      "execution_count": 162,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>Close_BTC</th>\n",
              "      <th>Volume_BTC</th>\n",
              "      <th>Number_of_trades_BTC</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2019-08-15 00:00:00</td>\n",
              "      <td>10144.54</td>\n",
              "      <td>114.720030</td>\n",
              "      <td>1459</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2019-08-15 00:01:00</td>\n",
              "      <td>10132.25</td>\n",
              "      <td>409.414904</td>\n",
              "      <td>3443</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2019-08-15 00:02:00</td>\n",
              "      <td>10132.19</td>\n",
              "      <td>118.248972</td>\n",
              "      <td>1028</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2019-08-15 00:03:00</td>\n",
              "      <td>10149.31</td>\n",
              "      <td>36.480215</td>\n",
              "      <td>422</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2019-08-15 00:04:00</td>\n",
              "      <td>10128.26</td>\n",
              "      <td>84.651933</td>\n",
              "      <td>747</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 date  Close_BTC  Volume_BTC  Number_of_trades_BTC\n",
              "0 2019-08-15 00:00:00   10144.54  114.720030                  1459\n",
              "1 2019-08-15 00:01:00   10132.25  409.414904                  3443\n",
              "2 2019-08-15 00:02:00   10132.19  118.248972                  1028\n",
              "3 2019-08-15 00:03:00   10149.31   36.480215                   422\n",
              "4 2019-08-15 00:04:00   10128.26   84.651933                   747"
            ]
          },
          "metadata": {},
          "execution_count": 162
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "kpM1pt_FHuL2",
        "outputId": "6a39940b-2b34-4ead-f4c8-4d38c990b2a2"
      },
      "source": [
        "#creation of new column named close_360 mins which represents the price of bitcoin(BTC) 6 hours prior to current time   \n",
        "df2['close_360min'] = df2.Close_BTC.shift(360)\n",
        "df2.head()"
      ],
      "execution_count": 163,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>Close_BTC</th>\n",
              "      <th>Volume_BTC</th>\n",
              "      <th>Number_of_trades_BTC</th>\n",
              "      <th>close_360min</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2019-08-15 00:00:00</td>\n",
              "      <td>10144.54</td>\n",
              "      <td>114.720030</td>\n",
              "      <td>1459</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2019-08-15 00:01:00</td>\n",
              "      <td>10132.25</td>\n",
              "      <td>409.414904</td>\n",
              "      <td>3443</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2019-08-15 00:02:00</td>\n",
              "      <td>10132.19</td>\n",
              "      <td>118.248972</td>\n",
              "      <td>1028</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2019-08-15 00:03:00</td>\n",
              "      <td>10149.31</td>\n",
              "      <td>36.480215</td>\n",
              "      <td>422</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2019-08-15 00:04:00</td>\n",
              "      <td>10128.26</td>\n",
              "      <td>84.651933</td>\n",
              "      <td>747</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 date  Close_BTC  ...  Number_of_trades_BTC  close_360min\n",
              "0 2019-08-15 00:00:00   10144.54  ...                  1459           NaN\n",
              "1 2019-08-15 00:01:00   10132.25  ...                  3443           NaN\n",
              "2 2019-08-15 00:02:00   10132.19  ...                  1028           NaN\n",
              "3 2019-08-15 00:03:00   10149.31  ...                   422           NaN\n",
              "4 2019-08-15 00:04:00   10128.26  ...                   747           NaN\n",
              "\n",
              "[5 rows x 5 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 163
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "l8hjRiLOH95q",
        "outputId": "55cef147-1c8d-4023-d383-434c740fec4c"
      },
      "source": [
        "#type conversion of column Timestamp as a pandas time series element \n",
        "df2['date'] =  pd.to_datetime(df2['date'], format='%d/%m/%Y %H:%M')\n",
        "df2.head()"
      ],
      "execution_count": 164,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>Close_BTC</th>\n",
              "      <th>Volume_BTC</th>\n",
              "      <th>Number_of_trades_BTC</th>\n",
              "      <th>close_360min</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2019-08-15 00:00:00</td>\n",
              "      <td>10144.54</td>\n",
              "      <td>114.720030</td>\n",
              "      <td>1459</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2019-08-15 00:01:00</td>\n",
              "      <td>10132.25</td>\n",
              "      <td>409.414904</td>\n",
              "      <td>3443</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2019-08-15 00:02:00</td>\n",
              "      <td>10132.19</td>\n",
              "      <td>118.248972</td>\n",
              "      <td>1028</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2019-08-15 00:03:00</td>\n",
              "      <td>10149.31</td>\n",
              "      <td>36.480215</td>\n",
              "      <td>422</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2019-08-15 00:04:00</td>\n",
              "      <td>10128.26</td>\n",
              "      <td>84.651933</td>\n",
              "      <td>747</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 date  Close_BTC  ...  Number_of_trades_BTC  close_360min\n",
              "0 2019-08-15 00:00:00   10144.54  ...                  1459           NaN\n",
              "1 2019-08-15 00:01:00   10132.25  ...                  3443           NaN\n",
              "2 2019-08-15 00:02:00   10132.19  ...                  1028           NaN\n",
              "3 2019-08-15 00:03:00   10149.31  ...                   422           NaN\n",
              "4 2019-08-15 00:04:00   10128.26  ...                   747           NaN\n",
              "\n",
              "[5 rows x 5 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 164
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "k2ELSsa5IBIR",
        "outputId": "593debac-b2de-4bc7-c1ad-6f705fefa1ae"
      },
      "source": [
        "#dropping row entries that contain null value. null values are created as we shift time stamp by 6 hours to determine lagged values \n",
        "df2.dropna(inplace=True)\n",
        "df2.head()"
      ],
      "execution_count": 165,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>Close_BTC</th>\n",
              "      <th>Volume_BTC</th>\n",
              "      <th>Number_of_trades_BTC</th>\n",
              "      <th>close_360min</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>360</th>\n",
              "      <td>2019-08-15 14:00:00</td>\n",
              "      <td>10034.93</td>\n",
              "      <td>32.579320</td>\n",
              "      <td>355</td>\n",
              "      <td>10144.54</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>361</th>\n",
              "      <td>2019-08-15 14:01:00</td>\n",
              "      <td>10063.10</td>\n",
              "      <td>34.863390</td>\n",
              "      <td>517</td>\n",
              "      <td>10132.25</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>362</th>\n",
              "      <td>2019-08-15 14:02:00</td>\n",
              "      <td>10065.28</td>\n",
              "      <td>83.969008</td>\n",
              "      <td>1088</td>\n",
              "      <td>10132.19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>363</th>\n",
              "      <td>2019-08-15 14:03:00</td>\n",
              "      <td>10051.01</td>\n",
              "      <td>47.185592</td>\n",
              "      <td>519</td>\n",
              "      <td>10149.31</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>364</th>\n",
              "      <td>2019-08-15 14:04:00</td>\n",
              "      <td>10085.67</td>\n",
              "      <td>28.420858</td>\n",
              "      <td>328</td>\n",
              "      <td>10128.26</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                   date  Close_BTC  ...  Number_of_trades_BTC  close_360min\n",
              "360 2019-08-15 14:00:00   10034.93  ...                   355      10144.54\n",
              "361 2019-08-15 14:01:00   10063.10  ...                   517      10132.25\n",
              "362 2019-08-15 14:02:00   10065.28  ...                  1088      10132.19\n",
              "363 2019-08-15 14:03:00   10051.01  ...                   519      10149.31\n",
              "364 2019-08-15 14:04:00   10085.67  ...                   328      10128.26\n",
              "\n",
              "[5 rows x 5 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 165
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "QqxY1CeVIDmd",
        "outputId": "51c46171-56d7-4d8c-f56d-f89f8588ffb9"
      },
      "source": [
        "#creating a column named diff_360min which is indicative of price difference between current timestamp and 6 hours prior \n",
        "df2['diff_360min']= df2['close_360min'] - df2['Close_BTC'] \n",
        "df2.head()"
      ],
      "execution_count": 166,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>Close_BTC</th>\n",
              "      <th>Volume_BTC</th>\n",
              "      <th>Number_of_trades_BTC</th>\n",
              "      <th>close_360min</th>\n",
              "      <th>diff_360min</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>360</th>\n",
              "      <td>2019-08-15 14:00:00</td>\n",
              "      <td>10034.93</td>\n",
              "      <td>32.579320</td>\n",
              "      <td>355</td>\n",
              "      <td>10144.54</td>\n",
              "      <td>109.61</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>361</th>\n",
              "      <td>2019-08-15 14:01:00</td>\n",
              "      <td>10063.10</td>\n",
              "      <td>34.863390</td>\n",
              "      <td>517</td>\n",
              "      <td>10132.25</td>\n",
              "      <td>69.15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>362</th>\n",
              "      <td>2019-08-15 14:02:00</td>\n",
              "      <td>10065.28</td>\n",
              "      <td>83.969008</td>\n",
              "      <td>1088</td>\n",
              "      <td>10132.19</td>\n",
              "      <td>66.91</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>363</th>\n",
              "      <td>2019-08-15 14:03:00</td>\n",
              "      <td>10051.01</td>\n",
              "      <td>47.185592</td>\n",
              "      <td>519</td>\n",
              "      <td>10149.31</td>\n",
              "      <td>98.30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>364</th>\n",
              "      <td>2019-08-15 14:04:00</td>\n",
              "      <td>10085.67</td>\n",
              "      <td>28.420858</td>\n",
              "      <td>328</td>\n",
              "      <td>10128.26</td>\n",
              "      <td>42.59</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                   date  Close_BTC  ...  close_360min  diff_360min\n",
              "360 2019-08-15 14:00:00   10034.93  ...      10144.54       109.61\n",
              "361 2019-08-15 14:01:00   10063.10  ...      10132.25        69.15\n",
              "362 2019-08-15 14:02:00   10065.28  ...      10132.19        66.91\n",
              "363 2019-08-15 14:03:00   10051.01  ...      10149.31        98.30\n",
              "364 2019-08-15 14:04:00   10085.67  ...      10128.26        42.59\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 166
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "gUXJAv1WIM6e",
        "outputId": "84eaf4e0-ddc1-46a1-83c4-2981cd985e5a"
      },
      "source": [
        "#we create a new column named indicator_360min which accounts for up/downswing in prices comparing present close price and close price 6 hours prior. Upswings are represented as '1' and downswings are represented as '0' \n",
        "df2['indicator_360min'] = np.where(df2['diff_360min'] > 0, 1, 0)\n",
        "df2.head()"
      ],
      "execution_count": 167,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>Close_BTC</th>\n",
              "      <th>Volume_BTC</th>\n",
              "      <th>Number_of_trades_BTC</th>\n",
              "      <th>close_360min</th>\n",
              "      <th>diff_360min</th>\n",
              "      <th>indicator_360min</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>360</th>\n",
              "      <td>2019-08-15 14:00:00</td>\n",
              "      <td>10034.93</td>\n",
              "      <td>32.579320</td>\n",
              "      <td>355</td>\n",
              "      <td>10144.54</td>\n",
              "      <td>109.61</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>361</th>\n",
              "      <td>2019-08-15 14:01:00</td>\n",
              "      <td>10063.10</td>\n",
              "      <td>34.863390</td>\n",
              "      <td>517</td>\n",
              "      <td>10132.25</td>\n",
              "      <td>69.15</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>362</th>\n",
              "      <td>2019-08-15 14:02:00</td>\n",
              "      <td>10065.28</td>\n",
              "      <td>83.969008</td>\n",
              "      <td>1088</td>\n",
              "      <td>10132.19</td>\n",
              "      <td>66.91</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>363</th>\n",
              "      <td>2019-08-15 14:03:00</td>\n",
              "      <td>10051.01</td>\n",
              "      <td>47.185592</td>\n",
              "      <td>519</td>\n",
              "      <td>10149.31</td>\n",
              "      <td>98.30</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>364</th>\n",
              "      <td>2019-08-15 14:04:00</td>\n",
              "      <td>10085.67</td>\n",
              "      <td>28.420858</td>\n",
              "      <td>328</td>\n",
              "      <td>10128.26</td>\n",
              "      <td>42.59</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                   date  Close_BTC  ...  diff_360min  indicator_360min\n",
              "360 2019-08-15 14:00:00   10034.93  ...       109.61                 1\n",
              "361 2019-08-15 14:01:00   10063.10  ...        69.15                 1\n",
              "362 2019-08-15 14:02:00   10065.28  ...        66.91                 1\n",
              "363 2019-08-15 14:03:00   10051.01  ...        98.30                 1\n",
              "364 2019-08-15 14:04:00   10085.67  ...        42.59                 1\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 167
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "JLj1ljHKIS7U",
        "outputId": "b2a24008-7fcc-439f-b94b-856bf947d9ac"
      },
      "source": [
        "df2 = df2.filter(items=['date', 'Close_BTC','Volume_BTC','Number_of_trades_BTC','indicator_360min'])\n",
        "df2.head()"
      ],
      "execution_count": 168,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>Close_BTC</th>\n",
              "      <th>Volume_BTC</th>\n",
              "      <th>Number_of_trades_BTC</th>\n",
              "      <th>indicator_360min</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>360</th>\n",
              "      <td>2019-08-15 14:00:00</td>\n",
              "      <td>10034.93</td>\n",
              "      <td>32.579320</td>\n",
              "      <td>355</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>361</th>\n",
              "      <td>2019-08-15 14:01:00</td>\n",
              "      <td>10063.10</td>\n",
              "      <td>34.863390</td>\n",
              "      <td>517</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>362</th>\n",
              "      <td>2019-08-15 14:02:00</td>\n",
              "      <td>10065.28</td>\n",
              "      <td>83.969008</td>\n",
              "      <td>1088</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>363</th>\n",
              "      <td>2019-08-15 14:03:00</td>\n",
              "      <td>10051.01</td>\n",
              "      <td>47.185592</td>\n",
              "      <td>519</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>364</th>\n",
              "      <td>2019-08-15 14:04:00</td>\n",
              "      <td>10085.67</td>\n",
              "      <td>28.420858</td>\n",
              "      <td>328</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                   date  Close_BTC  ...  Number_of_trades_BTC  indicator_360min\n",
              "360 2019-08-15 14:00:00   10034.93  ...                   355                 1\n",
              "361 2019-08-15 14:01:00   10063.10  ...                   517                 1\n",
              "362 2019-08-15 14:02:00   10065.28  ...                  1088                 1\n",
              "363 2019-08-15 14:03:00   10051.01  ...                   519                 1\n",
              "364 2019-08-15 14:04:00   10085.67  ...                   328                 1\n",
              "\n",
              "[5 rows x 5 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 168
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ocA49b4_IZ3O"
      },
      "source": [
        "#adding a constant value of 1 to fields'Voume_BTC', 'Number_of_trades_BTC' for smoothening \n",
        "df2['Volume_BTC'] = df2['Volume_BTC']+1\n",
        "df2['Number_of_trades_BTC'] = df2['Number_of_trades_BTC']+1"
      ],
      "execution_count": 169,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GeAkcxVWIcNp",
        "outputId": "3942f216-aed4-4e22-b0ff-2990102d6225"
      },
      "source": [
        "#merging numerical and tweet dataset, to create a final dataset which can then be used for analysis\n",
        "df = pd.merge(df2, df1, how='inner', on='date')\n",
        "df.shape"
      ],
      "execution_count": 170,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(27480, 6)"
            ]
          },
          "metadata": {},
          "execution_count": 170
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "y5YXRACCIeLw",
        "outputId": "8d71eaf3-5f4f-4ecf-fa4e-b97a6f0c98b2"
      },
      "source": [
        "#visualising dataset created for modelling\n",
        "df.head()"
      ],
      "execution_count": 171,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>Close_BTC</th>\n",
              "      <th>Volume_BTC</th>\n",
              "      <th>Number_of_trades_BTC</th>\n",
              "      <th>indicator_360min</th>\n",
              "      <th>text_feature</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2020-01-01 08:32:00</td>\n",
              "      <td>7190.57</td>\n",
              "      <td>12.043702</td>\n",
              "      <td>103</td>\n",
              "      <td>1</td>\n",
              "      <td>8.2644</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2020-01-01 11:30:00</td>\n",
              "      <td>7231.05</td>\n",
              "      <td>19.914479</td>\n",
              "      <td>276</td>\n",
              "      <td>0</td>\n",
              "      <td>11.6704</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2020-01-01 14:51:00</td>\n",
              "      <td>7221.50</td>\n",
              "      <td>18.900408</td>\n",
              "      <td>192</td>\n",
              "      <td>0</td>\n",
              "      <td>5773.61</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2020-01-01 15:29:00</td>\n",
              "      <td>7228.13</td>\n",
              "      <td>10.952466</td>\n",
              "      <td>125</td>\n",
              "      <td>0</td>\n",
              "      <td>10.9056</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2020-01-01 22:09:00</td>\n",
              "      <td>7219.85</td>\n",
              "      <td>10.854588</td>\n",
              "      <td>193</td>\n",
              "      <td>1</td>\n",
              "      <td>92.928</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 date  Close_BTC  ...  indicator_360min  text_feature\n",
              "0 2020-01-01 08:32:00    7190.57  ...                 1        8.2644\n",
              "1 2020-01-01 11:30:00    7231.05  ...                 0       11.6704\n",
              "2 2020-01-01 14:51:00    7221.50  ...                 0       5773.61\n",
              "3 2020-01-01 15:29:00    7228.13  ...                 0       10.9056\n",
              "4 2020-01-01 22:09:00    7219.85  ...                 1        92.928\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 171
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5o_iN1JkIhXy",
        "outputId": "42e4727e-b046-48fb-e13c-a1a29931b860"
      },
      "source": [
        "#check if ther there null values in our dataset\n",
        "df.isnull().sum()"
      ],
      "execution_count": 172,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "date                    0\n",
              "Close_BTC               0\n",
              "Volume_BTC              0\n",
              "Number_of_trades_BTC    0\n",
              "indicator_360min        0\n",
              "text_feature            0\n",
              "dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 172
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pYdOMVWdIjku",
        "outputId": "c15b7959-2873-466e-ffbb-a71e92809712"
      },
      "source": [
        "#check on value distribution of dependent variable, 'indicator_360min' if there exists an imbalance in dataset \n",
        "df['indicator_360min'].value_counts()"
      ],
      "execution_count": 173,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    15273\n",
              "1    12207\n",
              "Name: indicator_360min, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 173
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "2m0y0UMRImn_",
        "outputId": "6c448dcb-b048-4454-e85e-afe329f8df89"
      },
      "source": [
        "df=df[['date','Close_BTC','Volume_BTC','Number_of_trades_BTC','text_feature','indicator_360min']]\n",
        "df.head()"
      ],
      "execution_count": 174,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>Close_BTC</th>\n",
              "      <th>Volume_BTC</th>\n",
              "      <th>Number_of_trades_BTC</th>\n",
              "      <th>text_feature</th>\n",
              "      <th>indicator_360min</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2020-01-01 08:32:00</td>\n",
              "      <td>7190.57</td>\n",
              "      <td>12.043702</td>\n",
              "      <td>103</td>\n",
              "      <td>8.2644</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2020-01-01 11:30:00</td>\n",
              "      <td>7231.05</td>\n",
              "      <td>19.914479</td>\n",
              "      <td>276</td>\n",
              "      <td>11.6704</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2020-01-01 14:51:00</td>\n",
              "      <td>7221.50</td>\n",
              "      <td>18.900408</td>\n",
              "      <td>192</td>\n",
              "      <td>5773.61</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2020-01-01 15:29:00</td>\n",
              "      <td>7228.13</td>\n",
              "      <td>10.952466</td>\n",
              "      <td>125</td>\n",
              "      <td>10.9056</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2020-01-01 22:09:00</td>\n",
              "      <td>7219.85</td>\n",
              "      <td>10.854588</td>\n",
              "      <td>193</td>\n",
              "      <td>92.928</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 date  Close_BTC  ...  text_feature  indicator_360min\n",
              "0 2020-01-01 08:32:00    7190.57  ...        8.2644                 1\n",
              "1 2020-01-01 11:30:00    7231.05  ...       11.6704                 0\n",
              "2 2020-01-01 14:51:00    7221.50  ...       5773.61                 0\n",
              "3 2020-01-01 15:29:00    7228.13  ...       10.9056                 0\n",
              "4 2020-01-01 22:09:00    7219.85  ...        92.928                 1\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 174
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "jWf0SsQYIrP1",
        "outputId": "e326d751-1164-4369-be97-7b09113daa56"
      },
      "source": [
        "#extracting time metrics from timestamp, so as to be passed to model as parameters \n",
        "df['day'] = df['date'].dt.day\n",
        "df['month'] = df['date'].dt.month\n",
        "df['year'] = df['date'].dt.year\n",
        "df['hour']=df['date'].dt.hour\n",
        "df['minute']= df['date'].dt.minute\n",
        "df.head()"
      ],
      "execution_count": 175,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>Close_BTC</th>\n",
              "      <th>Volume_BTC</th>\n",
              "      <th>Number_of_trades_BTC</th>\n",
              "      <th>text_feature</th>\n",
              "      <th>indicator_360min</th>\n",
              "      <th>day</th>\n",
              "      <th>month</th>\n",
              "      <th>year</th>\n",
              "      <th>hour</th>\n",
              "      <th>minute</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2020-01-01 08:32:00</td>\n",
              "      <td>7190.57</td>\n",
              "      <td>12.043702</td>\n",
              "      <td>103</td>\n",
              "      <td>8.2644</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2020</td>\n",
              "      <td>8</td>\n",
              "      <td>32</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2020-01-01 11:30:00</td>\n",
              "      <td>7231.05</td>\n",
              "      <td>19.914479</td>\n",
              "      <td>276</td>\n",
              "      <td>11.6704</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2020</td>\n",
              "      <td>11</td>\n",
              "      <td>30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2020-01-01 14:51:00</td>\n",
              "      <td>7221.50</td>\n",
              "      <td>18.900408</td>\n",
              "      <td>192</td>\n",
              "      <td>5773.61</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2020</td>\n",
              "      <td>14</td>\n",
              "      <td>51</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2020-01-01 15:29:00</td>\n",
              "      <td>7228.13</td>\n",
              "      <td>10.952466</td>\n",
              "      <td>125</td>\n",
              "      <td>10.9056</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2020</td>\n",
              "      <td>15</td>\n",
              "      <td>29</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2020-01-01 22:09:00</td>\n",
              "      <td>7219.85</td>\n",
              "      <td>10.854588</td>\n",
              "      <td>193</td>\n",
              "      <td>92.928</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2020</td>\n",
              "      <td>22</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 date  Close_BTC  Volume_BTC  ...  year hour  minute\n",
              "0 2020-01-01 08:32:00    7190.57   12.043702  ...  2020    8      32\n",
              "1 2020-01-01 11:30:00    7231.05   19.914479  ...  2020   11      30\n",
              "2 2020-01-01 14:51:00    7221.50   18.900408  ...  2020   14      51\n",
              "3 2020-01-01 15:29:00    7228.13   10.952466  ...  2020   15      29\n",
              "4 2020-01-01 22:09:00    7219.85   10.854588  ...  2020   22       9\n",
              "\n",
              "[5 rows x 11 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 175
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "YKG32TCkIv6p",
        "outputId": "fd3f2fcd-907d-4bd4-c6ee-b4da7f05c573"
      },
      "source": [
        "#dropping column 'date', as its not required for further analysis\n",
        "df.drop(columns=['date'], inplace= True)\n",
        "df.head()"
      ],
      "execution_count": 176,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Close_BTC</th>\n",
              "      <th>Volume_BTC</th>\n",
              "      <th>Number_of_trades_BTC</th>\n",
              "      <th>text_feature</th>\n",
              "      <th>indicator_360min</th>\n",
              "      <th>day</th>\n",
              "      <th>month</th>\n",
              "      <th>year</th>\n",
              "      <th>hour</th>\n",
              "      <th>minute</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>7190.57</td>\n",
              "      <td>12.043702</td>\n",
              "      <td>103</td>\n",
              "      <td>8.2644</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2020</td>\n",
              "      <td>8</td>\n",
              "      <td>32</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>7231.05</td>\n",
              "      <td>19.914479</td>\n",
              "      <td>276</td>\n",
              "      <td>11.6704</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2020</td>\n",
              "      <td>11</td>\n",
              "      <td>30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>7221.50</td>\n",
              "      <td>18.900408</td>\n",
              "      <td>192</td>\n",
              "      <td>5773.61</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2020</td>\n",
              "      <td>14</td>\n",
              "      <td>51</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>7228.13</td>\n",
              "      <td>10.952466</td>\n",
              "      <td>125</td>\n",
              "      <td>10.9056</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2020</td>\n",
              "      <td>15</td>\n",
              "      <td>29</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>7219.85</td>\n",
              "      <td>10.854588</td>\n",
              "      <td>193</td>\n",
              "      <td>92.928</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2020</td>\n",
              "      <td>22</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Close_BTC  Volume_BTC  Number_of_trades_BTC  ...  year  hour  minute\n",
              "0    7190.57   12.043702                   103  ...  2020     8      32\n",
              "1    7231.05   19.914479                   276  ...  2020    11      30\n",
              "2    7221.50   18.900408                   192  ...  2020    14      51\n",
              "3    7228.13   10.952466                   125  ...  2020    15      29\n",
              "4    7219.85   10.854588                   193  ...  2020    22       9\n",
              "\n",
              "[5 rows x 10 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 176
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YGB-PgzrIyOk"
      },
      "source": [
        "#import library for creating train and test set\n",
        "from sklearn.model_selection import train_test_split"
      ],
      "execution_count": 177,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gmAHwVaUI0EI"
      },
      "source": [
        "#we create train and test set of ratio 0.8:0.2\n",
        "X_train, X_test, y_train, y_test = train_test_split(df.drop(columns=['indicator_360min']), df['indicator_360min'], test_size=0.2, random_state=42)"
      ],
      "execution_count": 178,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "awcgUkAOI19H",
        "outputId": "3366c78b-bdd6-4e51-dab2-af55da578dee"
      },
      "source": [
        "#exploring value distribution of dependent variable on train set \n",
        "y_train.value_counts()"
      ],
      "execution_count": 179,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    12152\n",
              "1     9832\n",
              "Name: indicator_360min, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 179
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jvj1ObsOI3y9",
        "outputId": "d6896dde-952f-4a60-a4ab-904c6e34346a"
      },
      "source": [
        "#exploring value distribution of dependent variable on test set \n",
        "y_test.value_counts()"
      ],
      "execution_count": 180,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    3121\n",
              "1    2375\n",
              "Name: indicator_360min, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 180
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nMq1TyvaI6cm"
      },
      "source": [
        "#importing ensemble model random forest classifier for modelling \n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "classifier=RandomForestClassifier()"
      ],
      "execution_count": 181,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LUc5IkjgI8U1",
        "outputId": "bd6bc86f-b669-4faf-d495-03a994cf6891"
      },
      "source": [
        "#fit train data \n",
        "classifier.fit(X_train,y_train)"
      ],
      "execution_count": 182,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n",
              "                       criterion='gini', max_depth=None, max_features='auto',\n",
              "                       max_leaf_nodes=None, max_samples=None,\n",
              "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
              "                       min_samples_leaf=1, min_samples_split=2,\n",
              "                       min_weight_fraction_leaf=0.0, n_estimators=100,\n",
              "                       n_jobs=None, oob_score=False, random_state=None,\n",
              "                       verbose=0, warm_start=False)"
            ]
          },
          "metadata": {},
          "execution_count": 182
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4t2VV19RI-T5"
      },
      "source": [
        "#import necessary libraries for cross validation\n",
        "from sklearn.model_selection import cross_val_score\n",
        "from sklearn.model_selection import RepeatedStratifiedKFold"
      ],
      "execution_count": 183,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eNPBJo65JAgX"
      },
      "source": [
        "#building an function to evaluate k_fold validation score\n",
        "def evaluate_model(X, y, model):\n",
        "    cv = RepeatedStratifiedKFold(n_splits=5, n_repeats=3, random_state=1)\n",
        "    scores = cross_val_score(model, X, y, scoring='accuracy', cv=cv, n_jobs=-1)\n",
        "    return scores"
      ],
      "execution_count": 184,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NSTihfPiJCa6"
      },
      "source": [
        "score_k_fold = evaluate_model(X_train, y_train, classifier)"
      ],
      "execution_count": 185,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mi67xSUzJEXE",
        "outputId": "9c788480-220d-402a-b274-123247fd2c18"
      },
      "source": [
        "score_k_fold"
      ],
      "execution_count": 186,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.77689334, 0.78371617, 0.78712759, 0.78189675, 0.78139217,\n",
              "       0.77439163, 0.77666591, 0.78371617, 0.78962929, 0.78730664,\n",
              "       0.77916761, 0.78348874, 0.7839436 , 0.77780305, 0.77979982])"
            ]
          },
          "metadata": {},
          "execution_count": 186
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sUppYKhHJGU6",
        "outputId": "aa8cd084-7a83-4d3d-aa4d-79a394e046dd"
      },
      "source": [
        "#computing mena of k_fold validation score \n",
        "score_k_fold.mean()"
      ],
      "execution_count": 187,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7817958986981295"
            ]
          },
          "metadata": {},
          "execution_count": 187
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tjM0nuIcJIxX"
      },
      "source": [
        "#predicting values of test set \n",
        "prediction=classifier.predict(X_test)"
      ],
      "execution_count": 188,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8ctrNiebJKZa",
        "outputId": "bfbda8af-df5f-47e4-c7b5-20b794825e8f"
      },
      "source": [
        "#classification report\n",
        "from sklearn.metrics import confusion_matrix,classification_report,accuracy_score\n",
        "print(confusion_matrix(y_test,prediction))\n",
        "print(accuracy_score(y_test,prediction))\n",
        "print(classification_report(y_test,prediction))"
      ],
      "execution_count": 189,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[2727  394]\n",
            " [ 640 1735]]\n",
            "0.8118631732168851\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.81      0.87      0.84      3121\n",
            "           1       0.81      0.73      0.77      2375\n",
            "\n",
            "    accuracy                           0.81      5496\n",
            "   macro avg       0.81      0.80      0.81      5496\n",
            "weighted avg       0.81      0.81      0.81      5496\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Tw1vojqBJOAf",
        "outputId": "f193cc72-5769-415e-e34c-1525bfce2b3f"
      },
      "source": [
        "#hyper parameter tuning ensemble model built \n",
        "from sklearn.model_selection import RandomizedSearchCV\n",
        "# Number of trees in random forest\n",
        "n_estimators = [int(x) for x in np.linspace(start = 200, stop = 2000, num = 10)]\n",
        "# Number of features to consider at every split\n",
        "max_features = ['auto', 'sqrt']\n",
        "# Maximum number of levels in tree\n",
        "max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]\n",
        "max_depth.append(None)\n",
        "# Minimum number of samples required to split a node\n",
        "min_samples_split = [2, 5, 10]\n",
        "# Minimum number of samples required at each leaf node\n",
        "min_samples_leaf = [1, 2, 4]\n",
        "# Method of selecting samples for training each tree\n",
        "bootstrap = [True, False]\n",
        "# Create the random grid\n",
        "random_grid = {'n_estimators': n_estimators,\n",
        "               'max_features': max_features,\n",
        "               'max_depth': max_depth,\n",
        "               'min_samples_split': min_samples_split,\n",
        "               'min_samples_leaf': min_samples_leaf,\n",
        "               'bootstrap': bootstrap}\n",
        "print(random_grid)"
      ],
      "execution_count": 190,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'n_estimators': [200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800, 2000], 'max_features': ['auto', 'sqrt'], 'max_depth': [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, None], 'min_samples_split': [2, 5, 10], 'min_samples_leaf': [1, 2, 4], 'bootstrap': [True, False]}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S3VZD7xZJX0t",
        "outputId": "7eb71a9c-cd87-4933-b1d3-56c7b279e256"
      },
      "source": [
        "# Use the random grid to search for best hyperparameters\n",
        "# First create the base model to tune\n",
        "rf = RandomForestClassifier()\n",
        "# Random search of parameters, using 3 fold cross validation, \n",
        "# search across 100 different combinations, and use all available cores\n",
        "rf_random = RandomizedSearchCV(estimator = rf, param_distributions = random_grid, n_iter = 100, cv = 3, verbose=2, random_state=42, n_jobs = -1)\n",
        "# Fit the random search model\n",
        "rf_random.fit(X_train, y_train)"
      ],
      "execution_count": 126,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fitting 3 folds for each of 100 candidates, totalling 300 fits\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n",
            "/usr/local/lib/python3.7/dist-packages/joblib/externals/loky/process_executor.py:691: UserWarning: A worker stopped while some jobs were given to the executor. This can be caused by a too short worker timeout or by a memory leak.\n",
            "  \"timeout or by a memory leak.\", UserWarning\n",
            "[Parallel(n_jobs=-1)]: Done  37 tasks      | elapsed: 21.3min\n",
            "[Parallel(n_jobs=-1)]: Done 158 tasks      | elapsed: 95.1min\n",
            "[Parallel(n_jobs=-1)]: Done 300 out of 300 | elapsed: 172.7min finished\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RandomizedSearchCV(cv=3, error_score=nan,\n",
              "                   estimator=RandomForestClassifier(bootstrap=True,\n",
              "                                                    ccp_alpha=0.0,\n",
              "                                                    class_weight=None,\n",
              "                                                    criterion='gini',\n",
              "                                                    max_depth=None,\n",
              "                                                    max_features='auto',\n",
              "                                                    max_leaf_nodes=None,\n",
              "                                                    max_samples=None,\n",
              "                                                    min_impurity_decrease=0.0,\n",
              "                                                    min_impurity_split=None,\n",
              "                                                    min_samples_leaf=1,\n",
              "                                                    min_samples_split=2,\n",
              "                                                    min_weight_fraction_leaf=0.0,\n",
              "                                                    n_estimators=100,\n",
              "                                                    n_jobs...\n",
              "                   param_distributions={'bootstrap': [True, False],\n",
              "                                        'max_depth': [10, 20, 30, 40, 50, 60,\n",
              "                                                      70, 80, 90, 100, 110,\n",
              "                                                      None],\n",
              "                                        'max_features': ['auto', 'sqrt'],\n",
              "                                        'min_samples_leaf': [1, 2, 4],\n",
              "                                        'min_samples_split': [2, 5, 10],\n",
              "                                        'n_estimators': [200, 400, 600, 800,\n",
              "                                                         1000, 1200, 1400, 1600,\n",
              "                                                         1800, 2000]},\n",
              "                   pre_dispatch='2*n_jobs', random_state=42, refit=True,\n",
              "                   return_train_score=False, scoring=None, verbose=2)"
            ]
          },
          "metadata": {},
          "execution_count": 126
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rN8LwuXsJbGj",
        "outputId": "f2b44e16-e134-49d6-8379-ca80e5e3ef6f"
      },
      "source": [
        "print(rf_random.best_params_)"
      ],
      "execution_count": 127,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'n_estimators': 1000, 'min_samples_split': 2, 'min_samples_leaf': 1, 'max_features': 'auto', 'max_depth': 50, 'bootstrap': False}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Feaj_FSeJbKJ"
      },
      "source": [
        "#building ensemble model using idetified best parameters\n",
        "bst_classifier=RandomForestClassifier(n_estimators=1000,min_samples_split=2,min_samples_leaf=1,max_depth=50, bootstrap= False)"
      ],
      "execution_count": 191,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "or4Cmx5MJgrw",
        "outputId": "d94aa53b-702b-4fe4-a0e4-f74758c7ee40"
      },
      "source": [
        "#performing cross valdiation\n",
        "score_k_fold = evaluate_model(X_train, y_train, bst_classifier)"
      ],
      "execution_count": 192,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/joblib/externals/loky/process_executor.py:691: UserWarning: A worker stopped while some jobs were given to the executor. This can be caused by a too short worker timeout or by a memory leak.\n",
            "  \"timeout or by a memory leak.\", UserWarning\n",
            "/usr/local/lib/python3.7/dist-packages/joblib/externals/loky/process_executor.py:691: UserWarning: A worker stopped while some jobs were given to the executor. This can be caused by a too short worker timeout or by a memory leak.\n",
            "  \"timeout or by a memory leak.\", UserWarning\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cDcLtStMJkVt",
        "outputId": "a36f7faa-e38c-49f4-e298-f163a75ace35"
      },
      "source": [
        "score_k_fold"
      ],
      "execution_count": 193,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.81896748, 0.8166932 , 0.82010462, 0.8207869 , 0.82165605,\n",
              "       0.80987037, 0.81646577, 0.81419149, 0.82306118, 0.82870792,\n",
              "       0.81964976, 0.82556288, 0.81464635, 0.81487378, 0.82120109])"
            ]
          },
          "metadata": {},
          "execution_count": 193
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w_kZnjM8JlGE",
        "outputId": "35bb731c-4930-4dec-c79b-969cc695b5cd"
      },
      "source": [
        "score_k_fold.mean()"
      ],
      "execution_count": 194,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8190959224480198"
            ]
          },
          "metadata": {},
          "execution_count": 194
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kiZYouhfJnc0",
        "outputId": "5ae58335-7ea6-4031-c198-fd0a50cb6198"
      },
      "source": [
        "#fit train data \n",
        "bst_classifier.fit(X_train,y_train)"
      ],
      "execution_count": 195,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RandomForestClassifier(bootstrap=False, ccp_alpha=0.0, class_weight=None,\n",
              "                       criterion='gini', max_depth=50, max_features='auto',\n",
              "                       max_leaf_nodes=None, max_samples=None,\n",
              "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
              "                       min_samples_leaf=1, min_samples_split=2,\n",
              "                       min_weight_fraction_leaf=0.0, n_estimators=1000,\n",
              "                       n_jobs=None, oob_score=False, random_state=None,\n",
              "                       verbose=0, warm_start=False)"
            ]
          },
          "metadata": {},
          "execution_count": 195
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YrhBkBFaJpxy"
      },
      "source": [
        "#predicting test set \n",
        "prediction=bst_classifier.predict(X_test)"
      ],
      "execution_count": 196,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XxprfG6uJryx",
        "outputId": "c0c20437-7e40-44b7-9e95-b3b875000d28"
      },
      "source": [
        "#classification report \n",
        "from sklearn.metrics import confusion_matrix,classification_report,accuracy_score\n",
        "print(confusion_matrix(y_test,prediction))\n",
        "print(accuracy_score(y_test,prediction))\n",
        "print(classification_report(y_test,prediction))"
      ],
      "execution_count": 197,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[2769  352]\n",
            " [ 474 1901]]\n",
            "0.8497088791848617\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.85      0.89      0.87      3121\n",
            "           1       0.84      0.80      0.82      2375\n",
            "\n",
            "    accuracy                           0.85      5496\n",
            "   macro avg       0.85      0.84      0.85      5496\n",
            "weighted avg       0.85      0.85      0.85      5496\n",
            "\n"
          ]
        }
      ]
    }
  ]
}