MachineLearning Projects/spam_filter_NPL.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "lttgVF-ObOBp"
   },
   "source": [
    "**Importing** **Libraries** "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "cPcTsbozG4Fj"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "W3guXvOnd9rz"
   },
   "source": [
    "**Importing Dataset**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "w5PkznLgHlBZ"
   },
   "outputs": [],
   "source": [
    "dataset=pd.read_csv('spam.csv',encoding='latin-1')\n",
    "x=dataset.iloc[:,[0]]\n",
    "y=dataset.iloc[:,[1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "0HTWGF0LIROT",
    "outputId": "7e838005-2054-4475-bc13-cc659a74cb9a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        v1\n",
      "0      ham\n",
      "1      ham\n",
      "2     spam\n",
      "3      ham\n",
      "4      ham\n",
      "...    ...\n",
      "5567  spam\n",
      "5568   ham\n",
      "5569   ham\n",
      "5570   ham\n",
      "5571   ham\n",
      "\n",
      "[5572 rows x 1 columns]\n"
     ]
    }
   ],
   "source": [
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "rZ93Eix7ITFy",
    "outputId": "19075f3c-afa8-4c30-f7e3-1fba16339615"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                     v2\n",
      "0     Go until jurong point, crazy.. Available only ...\n",
      "1                         Ok lar... Joking wif u oni...\n",
      "2     Free entry in 2 a wkly comp to win FA Cup fina...\n",
      "3     U dun say so early hor... U c already then say...\n",
      "4     Nah I don't think he goes to usf, he lives aro...\n",
      "...                                                 ...\n",
      "5567  This is the 2nd time we have tried 2 contact u...\n",
      "5568              Will Ì_ b going to esplanade fr home?\n",
      "5569  Pity, * was in mood for that. So...any other s...\n",
      "5570  The guy did some bitching but I acted like i'd...\n",
      "5571                         Rofl. Its true to its name\n",
      "\n",
      "[5572 rows x 1 columns]\n"
     ]
    }
   ],
   "source": [
    "print(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "l6RZ48VZb6FC"
   },
   "source": [
    "**Encoding into classification(ham(0) or spam(1) )**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "hk3kNdRIJX68",
    "outputId": "aba0a077-a363-4758-b9db-73ec68452657"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/sklearn/preprocessing/_label.py:251: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "lb=LabelEncoder()\n",
    "x=lb.fit_transform(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9POoo8lMMK0D",
    "outputId": "04823275-869e-4931-ac6b-3528056d1d32"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 1 ... 0 0 0]\n"
     ]
    }
   ],
   "source": [
    "print(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ugAgTWP5cZJJ"
   },
   "source": [
    "**Stemming**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ItdN6FNWNu4l",
    "outputId": "31fcac56-e12c-48f7-d7ca-ce68968ce8d5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import nltk\n",
    "nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.porter import PorterStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "aR28c4rdZCaM"
   },
   "outputs": [],
   "source": [
    "reply=[]\n",
    "for i in range(0,5572):\n",
    "  text=re.sub('[^a-zA-Z]',' ',dataset['v2'][i])\n",
    "  text=text.lower()\n",
    "  text=text.split()\n",
    "  \n",
    "  ps=PorterStemmer()\n",
    "  text=[ps.stem(word) for word in text if not word  in set(stopwords.words('english'))]\n",
    "  text=' '.join(text)\n",
    "  reply.append(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kEy1dTevbi1Z"
   },
   "source": [
    "**Converting the collection of texts to a matrix of token counts**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "FiweohErnaTJ"
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "cv=CountVectorizer(6200)\n",
    "x=cv.fit_transform(reply).toarray()\n",
    "y=dataset.iloc[:,[0]].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "EhAI-_7UeGeG"
   },
   "source": [
    "**Splitting dataset into train_set and test_set**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "QkwQd6yZUiLW"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "x_train,x_test,y_train,y_test=train_test_split(x,y,test_size=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "v-4PzSxMeEkJ"
   },
   "source": [
    "**Training the model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Ne4BFIS1U9M1",
    "outputId": "39a451d9-4c48-4d4a-f0c9-1cb0f98a610a"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/sklearn/naive_bayes.py:206: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GaussianNB(priors=None, var_smoothing=1e-09)"
      ]
     },
     "execution_count": 12,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.naive_bayes import GaussianNB\n",
    "classifier=GaussianNB()\n",
    "classifier.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "OQiLjymJcmg9"
   },
   "source": [
    "**Predicting the Test set results**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "MVhy725ZeCfX",
    "outputId": "d7729ac5-e0da-4363-a107-e90388f44524"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['ham' 'ham']\n",
      " ['ham' 'ham']\n",
      " ['ham' 'ham']\n",
      " ...\n",
      " ['spam' 'spam']\n",
      " ['spam' 'spam']\n",
      " ['ham' 'ham']]\n"
     ]
    }
   ],
   "source": [
    "y_pred=classifier.predict(x_test)\n",
    "print(np.concatenate( ( y_test.reshape(len(y_test),1),y_pred.reshape(len(y_pred),1) ) ,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "eWjLiy3DfUav",
    "outputId": "87917422-7d47-4b55-c07e-94ec077dd352"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[860 101]\n",
      " [ 25 129]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8869955156950673"
      ]
     },
     "execution_count": 14,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score,confusion_matrix\n",
    "cm=confusion_matrix(y_test,y_pred)\n",
    "print(cm)\n",
    "accuracy_score(y_test,y_pred)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "a8Evc4I8c8aB"
   },
   "source": [
    "**Predicting a (new)text**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "B3opGUhLqBEJ",
    "outputId": "41a3be24-89f7-43c6-f507-5a85474e37d9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['spam']\n"
     ]
    }
   ],
   "source": [
    "new_review = 'The IRS is Trying to Contact You'\n",
    "new_review = re.sub('[^a-zA-Z]', ' ', new_review)\n",
    "new_review = new_review.lower()\n",
    "new_review = new_review.split()\n",
    "\n",
    "ps = PorterStemmer()\n",
    "all_stopwords = stopwords.words('english')\n",
    "all_stopwords.remove('not')\n",
    "new_review = [ps.stem(word) for word in new_review if not word in set(all_stopwords)]\n",
    "new_review = ' '.join(new_review)\n",
    "new_corpus = [new_review]\n",
    "new_X_test = cv.transform(new_corpus).toarray()\n",
    "new_y_pred = classifier.predict(new_X_test)\n",
    "print(new_y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "rljhplavwERz"
   },
   "outputs": [],
   "source": [
    "# https://www.kaggle.com/uciml/zsms-spam-collection -- Dataset"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "spam_filter.py",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}