notebooks/PCA_Feature Eng.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "_active": false,
    "_cell_guid": "ca50d942-809f-21d9-aa30-7f4eb60999b3",
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Code for applying PCA to a dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_active": false,
    "_cell_guid": "cca36417-3de2-c57d-8616-10a6f0e65574",
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# General libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "#import PCA\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "#import graphing libraries\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "_active": false,
    "_cell_guid": "e98c7c95-b212-5060-ca03-d067e387fab8",
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['bathrooms', 'bedrooms', 'building_id', 'created', 'description', 'display_address', 'features', 'latitude', 'listing_id', 'longitude', 'manager_id', 'photos', 'price', 'street_address']\n"
     ]
    }
   ],
   "source": [
    "#load data into train_df and test_df for the transformation\n",
    "### presumably data will be pre-cleaned and features engineered prior to this \n",
    "### rather than reading straight from the source files\n",
    "### interest level should be pulled out\n",
    "train_df = pd.read_json(\"../Kaggle Files/train.json\")\n",
    "test_df = pd.read_json(\"../Kaggle Files/test.json\")\n",
    "\n",
    "# Set target and predictors\n",
    "target = 'interest_level'\n",
    "predictors = [c for c in train_df.columns if not c == target]\n",
    "\n",
    "print(predictors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Set up X_train and y_train\n",
    "X_train = train_df[predictors]\n",
    "y_train = train_df[[target]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#transform into normalized values if not already done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#perform PCA transformation\n",
    "pca = PCA()\n",
    "X_transformed = pca.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#plot scree plot of explained variance\n",
    "plt.plot(pca.explained_variance_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# determine number of components to keep\n",
    "num_comp = np.sum(np.cumsum(pca.explained_variance_ratio_) < .9)\n",
    "num_comp"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}