yelp_reviews_LR.ipynb

{
 "metadata": {
  "name": "yelp_reviews_LR"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "IMPORTS"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import os\n",
      "import json\n",
      "from itertools import islice\n",
      "import re\n",
      "from collections import Counter\n",
      "import pandas as pd\n",
      "import nltk\n",
      "print os.getcwd()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "/Users/ajrenold/Dropbox/iSchool/2013Spring/DataMining/yelp_project\n"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Get Academic Dataset Files"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "file_dict = {}\n",
      "file_dict['reviews'] = 'yelp_academic_dataset_review.json'\n",
      "file_dict['stopwords'] = 'stop-words-english3-google.txt'\n",
      "\n",
      "review_file = open(file_dict['reviews'])\n",
      "review_file_s = islice(review_file,10000)\n",
      "print review_file_s\n",
      "print file_dict\n",
      "\n",
      "    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "<itertools.islice object at 0x1134a5418>\n",
        "{'reviews': 'yelp_academic_dataset_review.json', 'stopwords': 'stop-words-english3-google.txt'}\n"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Load Stop_words file"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "f = open(file_dict['stopwords'])\n",
      "print f\n",
      "\n",
      "stops = {}\n",
      "\n",
      "for line in islice(f,None):\n",
      "    word = line.lower().strip()\n",
      "    if word not in stops:\n",
      "        stops[word] = True"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "<open file 'stop-words-english3-google.txt', mode 'r' at 0x11345ed20>\n"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Create Review DF"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "review_json = open(file_dict['reviews'])\n",
      "reviews_for_df = [ json.loads(line) for line in review_file_s ]\n",
      "review_df = pd.DataFrame(reviews_for_df)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "review_df\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 5,
       "text": [
        "<class 'pandas.core.frame.DataFrame'>\n",
        "Int64Index: 10000 entries, 0 to 9999\n",
        "Data columns:\n",
        "business_id    10000  non-null values\n",
        "date           10000  non-null values\n",
        "review_id      10000  non-null values\n",
        "stars          10000  non-null values\n",
        "text           10000  non-null values\n",
        "type           10000  non-null values\n",
        "user_id        10000  non-null values\n",
        "votes          10000  non-null values\n",
        "dtypes: int64(1), object(7)"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "review_df['business_id'].count()\n",
      "review_df['tokens'] = review_df['text'].apply(nltk.word_tokenize) # tokenize using nltk\n",
      "review_df['useful_votes'] = review_df['votes'].apply(lambda x:x['useful']) # extract useful votes\n",
      "review_df = review_df.set_index(['review_id']) # index on review_id\n",
      "\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "Counter(review_df['useful_votes'])\n",
      "review_df = review_df[review_df['useful_votes']>1] # 40 percent votes are 0 , 28 % are 1 , discarding those\n",
      "print review_df['useful_votes'].count()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "3022\n"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# training data set - for Norwig's algorithm\n",
      "\n",
      "import re, collections\n",
      "\n",
      "def words(text): return nltk.word_tokenize(text.lower())\n",
      "#return re.findall('[a-z]+', text.lower()) \n",
      "\n",
      "def train(features):\n",
      "    model = collections.defaultdict(lambda: 1)\n",
      "    for f in features:\n",
      "        model[f] += 1\n",
      "    return model\n",
      "\n",
      "NWORDS = train(words(file('big.txt').read()))\n",
      "\n",
      "alphabet = 'abcdefghijklmnopqrstuvwxyz'\n",
      "\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "\n",
      "def edits1(word):\n",
      "   splits     = [(word[:i], word[i:]) for i in range(len(word) + 1)]\n",
      "   deletes    = [a + b[1:] for a, b in splits if b]\n",
      "   transposes = [a + b[1] + b[0] + b[2:] for a, b in splits if len(b)>1]\n",
      "   replaces   = [a + c + b[1:] for a, b in splits for c in alphabet if b]\n",
      "   inserts    = [a + c + b     for a, b in splits for c in alphabet]\n",
      "   return set(deletes + transposes + replaces + inserts)\n",
      "\n",
      "def known_edits2(word):\n",
      "    return set(e2 for e1 in edits1(word) for e2 in edits1(e1) if e2 in NWORDS)\n",
      "\n",
      "def known(words): return set(w for w in words if w in NWORDS)\n",
      "\n",
      "def correct(word):\n",
      "    candidates = known([word]) or known(edits1(word)) or known_edits2(word) or [word]\n",
      "    #print candidates\n",
      "    return max(candidates, key=NWORDS.get)\n",
      "\n",
      "print correct('DizaSter')\n",
      "print correct('slooooooow')\n",
      "\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "DizaSter\n",
        "slooooooow"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def wordcorrect(inplist):\n",
      "    for i in range(len(inplist)-1):\n",
      "        \n",
      "        if inplist[i] not in NWORDS:\n",
      "            inplist[i] = correct(inplist[i].lower())    \n",
      "        else:\n",
      "            inplist[i] = inplist[i].lower()\n",
      "        \n",
      "        \n",
      "    return inplist\n",
      "\n",
      "test =  review_df['tokens']\n",
      "\n",
      "\n",
      "\n",
      "print type(nltk.word_tokenize(\"I did'nt want to go there\"))\n",
      "\n",
      "#review_df['tokens'] = review_df['tokens'].apply(wordcorrect)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "<type 'list'>\n"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "to create sparse matrix : \n",
      "http://scikit-learn.org/dev/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html\n",
      "\n",
      "to run regression : \n",
      "http://scikit-learn.org/dev/modules/generated/sklearn.linear_model.LinearRegression.html#sklearn.linear_model.LinearRegression.fit"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print stops"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'www': True, 'is': True, 'it': True, 'an': True, 'as': True, 'are': True, 'in': True, 'what': True, 'from': True, 'for': True, 'to': True, 'when': True, 'by': True, 'how': True, 'was': True, 'be': True, 'that': True, 'who': True, 'with': True, '\\xef\\xbb\\xbfi': True, 'a': True, 'on': True, 'about': True, 'this': True, 'of': True, 'will': True, 'where': True, 'the': True, 'com': True, 'or': True, 'at': True}\n"
       ]
      }
     ],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.feature_extraction.text import CountVectorizer\n",
      "from sklearn.feature_extraction.text import TfidfVectorizer\n",
      "import re\n",
      "#from sets import Set\n",
      "import numpy\n",
      "\n",
      "def tokenize(text):\n",
      "    text = re.sub(r\"[\\n\\.,;\\!\\?\\(\\)\\[\\]\\*/:+\\-\\~]\",\" \",text.lower())\n",
      "    text = re.sub(r\"(\\b[\\d]+\\b|\\b[\\d]+[a-z]+\\b)\",\" \",text)\n",
      "    #inplist = nltk.word_tokenize(text)    \n",
      "    inplist = text.split(' ')\n",
      "    finallist = list()\n",
      "    result = list()\n",
      "    # wordcorrect for tokens\n",
      "    #finallist = wordcorrect(inplist)\n",
      "    finallist = inplist\n",
      "    \n",
      "    for i in range(len(finallist)):        \n",
      "        if stops.has_key(inplist[i]): # remove stop words\n",
      "            continue\n",
      "        elif '$' in inplist[i]:\n",
      "            result.append('priceMention')\n",
      "        else:\n",
      "            result.append(inplist[i])\n",
      "    \n",
      "    return result\n",
      "\n",
      "extr_feat = list() # global extracted features list\n",
      "\n",
      "def tfidf_vectorize():\n",
      "    vectorizer1 = TfidfVectorizer(tokenizer = tokenize)\n",
      "    X1 = vectorizer1.fit_transform(review_df['text'][:3000]) # sparse matrix with tfidf weights\n",
      "    features = vectorizer1.get_feature_names()\n",
      "    #print features\n",
      "    #print vectorizer1.get_feature_names()\n",
      "    X1 = X1.toarray() # convert into 2d array \n",
      "    \n",
      "    for i in range(len(X1)):\n",
      "        \n",
      "        median = numpy.median(X1[i][np.nonzero(X1[i])]) # extract features with top 50 % tfidf weights\n",
      "        std = numpy.std(X1[i][np.nonzero(X1[i])])\n",
      "        for x in range(len(X1[i])):\n",
      "            if X1[i][x] >= median + (2*std):\n",
      "                #print features[x]\n",
      "                global extr_feat\n",
      "                try:\n",
      "                    if extr_feat.index(features[x]):\n",
      "                        continue\n",
      "                except ValueError:\n",
      "                    extr_feat.append(features[x])\n",
      "               \n",
      "         \n",
      "    #vectorizer = CountVectorizer(vocabulary = extr_feat)\n",
      "    #trnginp = vectorizer.fit_transform(corpus)\n",
      "    #print vectorizer.get_feature_names()\n",
      "    #print trnginp.toarray()\n",
      "    \n",
      "\n",
      "\n",
      "count_vect = None\n",
      "trnginp = None\n",
      "\n",
      "def count_vectorize():\n",
      "    global count_vect\n",
      "    global trnginp\n",
      "    global extr_feat\n",
      "    count_vect = CountVectorizer(tokenizer = tokenize,vocabulary = extr_feat, min_df=30)\n",
      "    trnginp = count_vect.fit_transform(review_df['text'][:3000])\n",
      "\n",
      "\n",
      "tfidf_vectorize()\n",
      "count_vectorize()\n",
      "\n",
      "#print result.ndim\n",
      "#print count_vect.vocabulary_\n",
      "#dir(trnginp)\n",
      "#print trnginp.shape\n",
      "#.get_feature_names()\n",
      "\n",
      "#count_vect = CountVectorizer(input = inpcorpus, tokenizer = tokenize)\n",
      "\n",
      "\n",
      "#vectop = count_vect.fit_transform(review_df['text'][:3])\n",
      "\n",
      "#cnt = 0\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 59
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#global extr_feat\n",
      "print trnginp.shape\n",
      "print extr_feat\n",
      "print len(extr_feat)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(3000, 2549)\n",
        "[u'', u'excellent', u'park', u'paths', u'entrees', u'mahi', u'cream', u'ice', u'pizza', u'and', u\"he's\", u'mad', u'pics', u'lunch', u'i', u'prefer', u'type', u'afford', u'clothes', u'stuff', u'burger', u'cheese', u'pile', u'deep', u'quesadillas', u'japanese', u'check', u'waffles', u'she', u'strongbow', u'margaritas', u'pho', u'gift', u'range', u'body', u'yoga', u'bar', u'hockey', u'chic', u'nails', u'pedicures', u'skirt', u'before', u'min', u'rolled', u'yellow', u'oily', u'dollar', u'room', u'smoking', u'always', u'seem', u'sushi', u'airports', u'pet', u'manicure', u\"it's\", u'indian', u'london', u'anymore', u'beans', u'green', u'products', u'lager', u'hippies', u'come', u'portions', u'dos', u'gringos', u'painted', u'crab', u'slices', u'gyro', u'founder', u'hang', u'starbucks', u'writing', u'classes', u'wheat', u'coffee', u\"aj's\", u'fennel', u'grocer', u'merc', u'brittle', u'cous', u'muster', u'full', u'we', u'baby', u'brat', u'suggest', u'pupusas', u'tempe', u'location', u'he', u'thai', u'tom', u'waitress', u'ballet', u'nutcracker', u'performance', u'pie', u'pies', u'gammage', u'pass', u'performances', u'seats', u'desperation', u'medal', u'china', u'shrimp', u'steakhouse', u'tower', u'meh', u'goodwill', u'blah', u'crew', u'sidebar', u'soup', u'mmmm', u'trucks', u'amber', u'date', u'glaze', u'breakfast', u'camelback', u'cotton', u'inn', u\"rita's\", u'sublime', u'fill', u't', u'chez', u'nous', u'corn', u'cold', u'french', u'toast', u'mushroom', u'says', u'goat', u'olives', u'breasts', u'trusted', u'turkey', u'monroe', u'valuable', u'cute', u'hug', u'seat', u'salad', u'ribs', u'spinach', u'powder', u'art', u'supply', u'bibimbap', u'yummy', u'delivery', u'boba', u'milk', u'eggplant', u'very', u'ramen', u'w', u'veggies', u'elephant', u'studio', u'bros', u'cabana', u'resort', u'you', u'facilities', u'industry', u'massage', u'my', u'spa', u'therapist', u'guac', u'coupon', u'\"hot', u'burger\"', u'hoegaarden', u'event', u'francis', u'our', u'shall', u'carded', u'brunch', u'bungalow', u'works', u'bake', u'than', u'campus', u'level', u'yelp', u'nyc', u'philly', u'barrio', u'flan', u'humble', u'just', u'mediterranean', u\"zoe's\", u'chip', u'key', u'lime', u'definite', u'visits', u'dread', u'pain', u'hacienda', u'\"', u'jim', u'reported', u'coconut', u'chai', u'echo', u'set', u'show', u'shoes', u'india', u'add', u'kiltlifter', u\"neiman's\", u'dietary', u'restrictions', u\"haven't\", u'eat', u'here', u'fez', u'gas', u'station', u'peak', u'piestewa', u\"culver's\", u'midwest', u'wisconsin', u'beds', u'bored', u\"you'll\", u'fried', u'garlic', u\"i'd\", u'katsu', u'brisket', u'empty', u'enchilada', u'pretty', u'\"in', u'place\"', u'chinese', u'films', u'video', u'carrots', u'couscous', u'&', u'mgr', u'register', u'retail', u\"whitey's\", 'priceMention', u'randy', u'market', u'smoothie', u'ho', u'hotels', u'palms', u'royal', u'quiche', u'cocktails', u'risk', u'grinders', u'mornings', u'they', u'wearing', u'fnb', u'pavle', u'mai', u'tai', u'hair', u'laser', u'hikers', u'medical', u'supplies', u'trail', u'thrift', u'dance', u'lesson', u'lessons', u'hella', u'highway', u'worth', u'horrible', u'popcorn', u'fe', u'santa', u'thirsty', u'water', u'carpaccio', u'blast', u'cool', u'dancing', u'waffle', u'learn', u'class', u'grease', u'greasy', u'salt', u'margarita', u'shoe', u'asada', u'carne', u'boiled', u'fries', u'review', u'sriracha', u'dog', u'ringo', u'cupcake', u'sprinkles', u'witty', u'clothing', u'bf', u'meatballs', u'pickled', u'isle', u'noodles', u'cpk', u'drift', u'round', u'shots', u'mesa', u'runs', u'service', u'camping', u'wilderness', u'hike', u'mormon', u'river', u'tube', u'aspect', u'hoa', u'shank', u'dates', u'designer', u'nordstrom', u'curry', u'more', u'truth', u'want', u'mrs', u\"white's\", u'bye', u'counter', u'panini', u'icecream', u'republic', u\"kid's\", u'pasta', u'fifth', u'floor', u'stayed', u'miracle', u'mold', u'maintained', u'accessories', u'ikea', u'kinds', u'basic', u'line', u'bathroom', u'oil', u'sympathetic', u'lobster', u'tiramisu', u'pitcher', u'pool', u'kierland', u'phoenix', u'resorts', u'during', u'baba', u'haji', u'sales', u'switch', u'papaya', u\"wendy's\", u'willo', u'mac', u\"defalco's\", u'cheap', u'heading', u'camera', u'amazing', u'feedback', u'sunday', u'tea', u'wall', u'giggle', u'pink', u'spot', u'cartel', u'rules', u'pcg', u'custom', u'reuben', u'gate', u'cleanliness', u'vada', u'app', u'cut', u'orange', u'amore', u'gelato', u'rob', u'fitness', u'programs', u'generally', u'boring', u'sodas', u'call', u'phone', u'brow', u'chairs', u'gross', u'tastes', u'person', u'pasty', u'foxy', u'karaoke', u'private', u'gym', u'kids', u'northwest', u'ribbon', u'guests', u'classy', u\"hanny's\", u'pbs', u'told', u'daughter', u'miso', u'movie', u'biscuits', u'gravy', u'downfall', u'beer', u'game', u'their', u'cant', u'mock', u'tofu', u'exchange', u\"joe's\", u'fish', u'#', u'heart', u'made', u'bikes', u'taco', u'platter', u'pei', u'wei', u'sliders', u\"farmer's\", u'sourdough', u'bandido', u'chino', u'years', u'child', u'teachers', u'healthy', u'sounded', u'equipment', u'rent', u'buffalo', u'credit', u'items', u'sacks', u'mojito', u'camarones', u'enchiladas', u'pad', u'whoop', u'filipino', u'halo', u'cough', u'tamales', u'vegan', u'sandwiches', u'chilaquiles', u'chorizo', u'gallo', u'jj', u'volcano', u'jamaican', u'chalk', u'selections', u'boat', u'lake', u'eastern', u'hummus', u'plate', u'vegetarian', u'consistently', u'solid', u'animals', u'harley', u'home', u'humane', u'me', u'new', u'society', u'vicinity', u'hole', u'rock', u'sculpted', u'south', u'boulders', u'eba', u'theatre', u'ufc', u'raw', u'traveling', u'sub', u'blueberry', u'compote', u'minutes', u'pancakes', u'firesky', u'pizzas', u\"moore's\", u'him', u'fresh', u'guacamole', u'tacos', u'car', u'hiked', u'mountain', u\"they've\", u'breweries', u'ish', u'chocolate', u'red', u'velvet', u\"o'clock\", u'city', u'culture', u'lunches', u'potato', u\"arriba's\", u'chile', u'mexico', u'everyone', u'omelet', u'theme', u'rocks', u\"maizie's\", u'vet', u'dobbins', u'lookout', u\"chloe's\", u'mole', u'alcohol', u'visit', u'mill', u\"carl's\", u'english', u'speak', u'early', u'cupcakes', u'tickets', u'barber', u'david', u'haircut', u'happy', u\"papago's\", u'pannini', u'chocolately', u'roommate', u'wonderfulness', u'steak', u'bill', u'card', u'returned', u'honestly', u'choosing', u'deliciously', u'spicey', u'mark', u'taylor', u'takeout', u'worried', u'customer', u'great', u'sports', u'superbowl', u'dvds', u'conditioned', u'latte', u'mj', u'lb', u'shell', u'smokey', u'sake', u'hotel', u'lounge', u'parking', u'apartments', u'broke', u'closet', u'out', u'pomegranate', u'country', u'bacon', u'entree', u'lettuce', u'lot', u'played', u'sick', u'flops', u\"devil's\", u'unlimited', u'wash', u'washes', u'floats', u'pastrami', u'company', u'estimate', u'insurance', u'nosh', u'blossom', u'papago', u'didnt', u'wrap', u'truck', u'wraps', u'bbq', u'disappointment', u'positive', u'flour', u'handmade', u'tortillas', u'changes', u'had', u'trunk', u'peking', u'beppo', u'buca', u'di', u'italian', u'hungry', u'sandwich', u'valet', u'\"no', u'guy', u'slice', u'mint', u'abysmal', u'bomberos', u'print', u'prado', u'many', u'rather', u'after', u\"couldn't\", u'drunken', u'its', u'sis', u'split', u'somehow', u'york', u'asian', u'chimi', u'heed', u'mexican', u'unsure', u'tire', u'tires', u'minnie', u'wag', u'beef', u'fail', u'gyros', u'central', u'bands', u'entry', u'marquee', u'problems', u'frozen', u'puppy', u'yogurt', u'respected', u'commission', u'im', u'online', u'sign', u'sore', u'champagne', u'yum', u'clearance', u'dillards', u'fashion', u\"macy's\", u'square', u'records', u'chestnut', u'lane', u'order', u'ours', u'frou', u'satay', u'falafel', u'greek', u'loud', u'rula', u'meatball', u'partner', u'beaten', u'thier', u'vacuum', u'website', u'cleaning', u'coal', u'crust', u'fired', u\"grimaldi's\", u'oven', u'cheesecake', u'cousin', u'factory', u'band', u'postinos', u\"'n'\", u'chips', u'trivia', u'anxious', u'kitsch', u'two', u'marshmallow', u'blanket', u'screen', u'attractive', u'documentary', u'lentil', u'loader', u'oysters', u'seafood', u'edit', u'tostada', u'groupon', u'texas', u'auto', u'scott', u'blanco', u'showed', u'bianco', u'bob', u'chris', u'pizzeria', u'club', u'german', u'yelpers', u'la', u'players', u'donut', u'donuts', u'chill', u'frank', u'free', u'sorbet', u'love', u'rushed', u'metal', u'max', u'office', u'jalapeno', u'monday', u'nowhere', u'dive', u'gear', u'shortcake', u'sister', u'alley', u'sashimi', u'arcadia', u'mission', u'chalet', u'comedy', u'despite', u'le', u'fritters', u'sweet', u'moist', u'cakes', u'coe', u'tammie', u'salsas', u'cooking', u'fantastic', u'lee', u'did', u'chef', u'purple', u'wax', u'gf', u'plantains', u'punch', u'rum', u\"pomeroy's\", u'purpose', u'relationship', u'friend', u'sophia', u'empanadas', u'account', u'checking', u'fees', u'salsa', u'tamale', u'salon', u'savings', u'pedicure', u'walmart', u'go', u'us', u'dim', u'sum', u'booths', u'chicken', u'pita', u'paul', u'unforgettable', u'luna', u'valle', u'ignite', u'tap', u'gold', u'amateur', u'roots', u'sauces', u'robin', u\"taylor's\", u'douche', u'y', u'hh', u\"pete's\", u'participating', u'eddie', u'mayo', u'croissant', u'doubt', u'essence', u'morning', u'particularly', u'pastry', u'pudding', u'vanilla', u'cats', u'dogs', u'pickles', u'please', u'dozen', u'grill', u'grits', u'traffic', u'entertainment', u'chu', u'johnny', u'sens', u'tapas', u\"rico's\", u'job', u'hour', u'cake', u'honey', u'moon', u'sweets', u'mall', u\"there's\", u'knife', u'knives', u'shake', u'son', u'sauce', u'ra', u'shirts', u'peanut', u'play', u'bosa', u'glazed', u'duck', u'fat', u'wait', u'quesadilla', u'audition', u'people', u'nights', u'stage', u'fb', u'airport', u'flirting', u'dough', u'fabric', u'peter', u'piper', u'miss', u'neglected', u'tubes', u'wheel', u'bulgogi', u'tokyo', u'windows', u'cherryblossom', u'crudo', u'bratwurst', u'brewery', u'light', u'rail', u'consult', u'facility', u'his', u'trust', u'automotive', u'quiznos', u'rio', u'bagel', u'accent', u'burgers', u'spaghetti', u'incompetent', u'persuade', u'espresso', u'don', u'tuck', u'either', u'boo', u'carriers', u'southwest', u'mini', u'\"white\"', u\"chelsea's\", u'dj', u'death', u'f', u'p', u'bone', u'rudys', u'today', u'understaffed', u'blow', u'pools', u'mins', u'mu', u'shu', u'cantina', u'hue', u'her', u'golden', u'spoon', u'andrea', u\"she's\", u'oatmeal', u'minced', u'salary', u'zucca', u'airlines', u'flight', u'party', u'lux', u'ground', u'meat', u'depends', u'lloyd', u\"they're\", u'cookie', u'lgo', u'jazz', u'music', u'downstairs', u'there', u'upstairs', u'z', u'roll', u'bento', u\"we'll\", u'awesome', u'shawarma', u\"fleming's\", u'salmon', u'foo', u'toffee', u'trainers', u\"doesn't\", u'star', u'cans', u'front', u'trash', u\"richardson's\", u'specials', u'shopper', u'costco', u'snack', u'busboy', u'were', u'kimchi', u'seasoning', u'used', u'groomer', u'gays', u'da', u'vang', u'course', u'biz_photos', u'http', u'eyebrow', u'blue', u'caramel', u'salted', u'sundae', u'crisp', u'curd', u'weather', u'feeling', u'cab', u'strong', u'visited', u'chili', u'meet', u'amy', u'ganoush', u'loooove', u'carrying', u'bikini', u'bum', u'might', u'marked', u'scratches', u'butter', u'opposed', u'bruschetta', u'fight', u'prepare', u'vino', u'rude', u'workers', u'okay', u'harbor', u'sky', u'neighborhood', u'skirts', u'flakes', u'cup', u'one', u'theater', u'sophomore', u'suede', u'mekong', u\"macalpine's\", u'ordered', u'partly', u'dvd', u'netflix', u'package', u'bowling', u'refreshing', u'joe', u'liberty', u'bra', u'christopher', u'refund', u'cc', u'liars', u'mean', u'came', u'cooked', u'medium', u'dine', u'gilbert', u'beers', u'brewing', u'porter', u'buffet', u'videos', u\"wasn't\", u'scare', u'cement', u'circle', u'grills', u'peoria', u'picnic', u'ramadas', u'glendale', u'jungle', u'locations', u'canal', u'pig', u'packed', u'treatments', u'skating', u'showing', u'sox', u'cibo', u'venue', u'wedding', u'ethiopian', u'burrito', u'mussels', u'dr', u'breading', u'miners', u'op', u'adobo', u\"karey's\", u'lumpia', u'rice', u'pork', u'shows', u'\"that', u\"'s\", u'arrowhead', u'malls', u'il', u'broken', u'bowl', u'selected', u'league', u'spring', u'candy', u'yourself', u'chance', u'update', u'grandma', u'iced', u'aisle', u'cart', u'carts', u'f&e', u'grocery', u'group', u'naked', u'turd', u'mats', u'figure', u'myst', u\"'oh\", u'employees', u'business', u'banh', u'woulda', u'audio', u'library', u'corned', u'jewish', u'seating', u'your', u'downtown', u'porch', u'box', u'produce', u'heavenly', u'stock', u'mash', u'prior', u'contacts', u\"barro's\", u'creamed', u'chin', u'portabello', u'dd', u\"rudy's\", u'havana', u'muscles', u'paella', u'buying', u'maple', u'citizen', u'torta', u'bangers', u'irish', u'sicilian', u'girlfriend', u'changing', u'lately', u'peaks', u'generous', u'ya', u'tuesday', u'apparel', u'scissors', u'up', u'peach', u'spicy', u'tempura', u'milkshake', u'fig', u'lox', u'mascarpone', u'links', u'interior', u'minus', u'forbidden', u'gay', u'deli', u'steaks', u'boutique', u'rotisserie', u'spread', u'friday', u'hip', u'hop', u'fry', u'per', u'noise', u'gal', u'press', u'quiet', u'client', u'available', u'chops', u'product', u'ritz', u'patatas', u'charcoal', u'blood', u'fucking', u'kathy', u'takes', u'halibut', u'fun', u'property', u'baseball', u'convention', u'stadium', u'candle', u'sober', u'delta', u\"isn't\", u'seems', u'mocha', u'mom', u'olive', u'brioche', u'burnt', u'onion', u'straws', u'thin', u'kisra', u'hefeweisen', u'roast', u'grape', u'activity', u'acts', u'dbg', u'festive', u'garden', u'luminarias', u'bottle', u'backyard', u'oleanders', u'protein', u'o', u'open', u'christine', u'technician', u'ride', u'ann', u'ribeye', u'roadhouse', u'culinary', u'greatest', u'jar', u'co', u'vermont', u'j', u'amc', u'harkins', u'mama', u'adult', u'am', u'cleaned', u'scheduled', u'thrilled', u'women', u'really', u'companions', u'jamba', u'smoothies', u'dirty', u'easy', u'relatively', u'cultural', u'foods', u'hong', u'inexpensive', u'kong', u'plaza', u'item', u'savers', u'tag', u'\"what', u'air', u'hands', u'raise', u'lengua', u'foothills', u'healthcare', u'hospital', u'hospitals', u\"i'll\", u\"i've\", u\"rubio's\", u'swimming', u'greece', u\"al's\", u'chicago', u'jus', u'pulling', u'decisions', u'android', u'at&t', u'feelings', u'store', u'large', u'dissatisfied', u'ultra', u'warranted', u'important', u'chambord', u\"houston's\", u'construction', u'q', u'hot', u'usually', u'creation', u'offerings', u'biltmore', u'angel', u'pricier', u'hob', u\"nob's\", u'smoke', u'poser', u'second', u'shaving', u'executed', u'shampoo', u'wound', u'spin', u'outdoor', u'cobbler', u'magical', u'repair', u'keys', u'machine', u'pants', u'thanh', u'tight', u'football', u'center', u'wine', u'guided', u'heather', u'foodies', u'rosemary', u'flavorless', u'get', u'draft', u'pretzel', u'pretzels', u'not', u'times', u'blossoms', u'squash', u'stringy', u'cha', u'drive', u'needles', u'road', u'hash', u'side', u'blanc', u'ferrari', u'alice', u'land', u'public', u'hut', u'roof', u'tuna', u'algae', u'infested', u'led', u'tank', u'tanks', u'jen', u'mushrooms', u'vietnamese', u'crumbled', u'ravioli', u'postino', u'impatient', u'skills', u'amanda', u'habanero', u'remove', u'silverware', u'jon', u'kinda', u'burned', u'calzone', u'napa', u'newspaper', u'gut', u'appraisal', u'ring', u'beet', u'fee', u'\"last', u'pair', u'even', u'nom', u'server', u'quote', u\"christopher's\", u'byob', u'answering', u'ipa', u'throwing', u'pear', u'authentic', u'de', u'tavern', u'tour', u'refills', u'urban', u'omelette', u'families', u'stocked', u'rings', u\"ted's\", u'toilet', u'valve', u'gifts', u'merchandise', u'e', u'mailed', u'heavily', u'chop', u\"matt's\", u'pond', u'custard', u'bucks', u'costume', u'halloween', u'roost', u'shumai', u'adovada', u'treadmill', u'designated', u'performers', u'chew', u'corral', u'dessert', u'sizzler', u'local', u'aggressive', u\"let's\", u'biryani', u'rnr', u'arepas', u'giants', u'series', u'tvs', u'eggs', u'money', u'sing', u'singing', u'crispy', u'food', u'kept', u'since', u'disco', u'sharpened', u\"cabela's\", u'whatsoever', u'including', u'creams', u'tell', u'area', u'surgery', u'magazine', u'wallpaper', u'license', u'picture', u'state', u'ocean', u'hangout', u'ny', u'western', u'change', u'terrible', u'husband', u'section', u'throne', u'commons', u'independent', u'everybody', u'pupusa', u'salvadoreno', u'kalbi', u'korean', u'\"real\"', u'work', u'yard', u'freaking', u'sucks', u'catfish', u'rama', u'self', u'mousse', u'nachos', u'pee', u'farm', u\"yc's\", u'behavior', u'driver', u'wrong', u'mcgrath', u'closest', u'buffets', u'delux', u\"father's\", u'ones', u'books', u'clerk', u'gallina', u'magic', u'fatburger', u'thanksgiving', u'bread', u'jimmy', u'quieter', u'therefore', u'mam', u'nuoc', u'parmigiano', u'reggiano', u'pawn', u'pick', u'window', u'doctor', u'\\r', u'pepperoni', u'searched', u'carpet', u'tree', u'kung', u'larges', u'niece', u'jerk', u'roti', u'guinness', u'jameson', u'viet', u'exude', u'tables', u'waited', u'denver', u'fair', u'steady', u'trade', u'coworker', u'quickly', u'lemon', u'firecracker', u'argued', u'cookies', u'ethnic', u'fiance', u'noca', u'alternative', u'think', u'lighting', u'loved', u'pop', u'soda', u\"lolo's\", u'poop', u'drunk', u'community', u'diamond', u'lines', u'began', u'brush', u'roller', u'warranty', u'affliction', u'bro', u'crown', u'rose', u'fancy', u'disliked', u'disposable', u'explained', u'non', u'cycle', u'deal', u'basil', u'better', u'hilton', u'sale', u'ski', u'guilt', u'rancheros', u'saved', u'contacted', u'cheesesteaks', u'freakin', u'unhappy', u'minimum', u'live', u'nightcap', u'aerobics', u'because', u'swim', u'vintage', u'mistake', u'sure', u'good', u'atrium', u'decide', u'tart', u'scone', u\"durant's\", u'trout', u'bucco', u'osso', u'outdated', u'ah', u'cauliflower', u'justify', u'would', u'casino', u'poker', u'bang', u'compensation', u'trainer', u'wife', u'suburbs', u'vitamin', u'fajitas', u'bus', u'chick', u'fast', u'fil', u\"mickey's\", u'doors', u'lady', u'mucho', u'numero', u'realizing', u'vegas', u'apple', u'smoked', u'rays', u'casita', u'villas', u'mamma', u'mia', u'machaca', u'birthday', u'color', u'stylist', u'sink', u'villa', u'mcdowell', u'b', u'c', u'book', u'bistro', u'france', u'paris', u'wee', u'karen', u'rental', u'basics', u'racial', u'slurs', u'bliss', u'festival', u'rainbow', u'mealy', u'tomatoes', u'instructors', u'student', u'studios', u'throughout', u'fan', u'montelucia', u'dressed', u'gimmick', u'lard', u'ordinary', u\"felipe's\", u'san', u'movies', u'weds', u'munch', u'scenery', u'dealership', u'jeep', u'bollywood', u'panda', u'fred', u'degree', u'design', u'students', u'suggested', u'avenue', u'hospitality', u'laundry', u'suite', u'presentation', u'scallop', u'surprisingly', u'greeted', u'interestingly', u'd', u'pickle', u'lo', u\"lo's\", u'frites', u'lindsay', u'milano', u'radio', u'ants', u'hubster', u'carnitas', u'tortas', u'activities', u'bed', u'site', u'canned', u'grande', u'seated', u'cocktail', u'school', u'cardinal', u'pm', u'skip', u'kid', u'balcony', u'bizarre', u'botanical', u'cacti', u'gardens', u'plants', u'nobuo', u'teeter', u'jacuzzi', u'bc', u'hi', u'settings', u'dress', u'silk', u'homophobia', u'pre', u'boys', u'esplanade', u'theaters', u'candles', u'shelves', u'dairy', u'guessing', u'carry', u'four', u'scottish', u'steve', u'flew', u'lola', u'hours', u'wimpy', u'children', u'cadillac', u'seabass', u'mother', u'say', u'saturdays', u'union', u'fave', u'staffer', u'remember', u'teacher', u'active', u'wasnt', u'sprouts', u'breve', u'thru', u'gryo', u'platters', u'spanakopita', u'bath', u'kimpton', u'awhatukee', u\"one's\", u'paleta', u'strawberry', u'disgusting', u'tip', u'voucher', u'ken', u'pricing', u'keeping', u'reservation', u'screens', u'them', u'bin', u'bragging', u'john', u'owner', u'shop', u'watch', u'name', u\"would've\", u'written', u'envy', u'nearly', u'vegans', u'brunswick', u'cuban', u'average', u'said', u'some', u'system', u'alligator', u'feta', u'tongue', u'exciting', u'burritos', u'ahi', u'but', u'herb', u'lentils', u'drinkers', u'juicy', u'nauseous', u'rides', u'scent', u'sipped', u'owners', u'helpful', u'midst', u'mimosas', u'tint', u'several', u'table', u'fed', u'organic', u\"arby's\", u'ff', u'overflow', u'blase', u'allot', u'snarky', u'anyone', u'wings', u'zipps', u'ladies', u'door', u'salty', u'wifi', u'tanked', u'traditional', u'casserole', u'pea', u'others', u'rest', u'usual', u'hillstone', u\"roy's\", u'black', u'acquired', u'safeway', u'shopping', u'huge', u'select=', u'sail', u'asu', u'skillet', u\"year's\", u'howard', u'changed', u'view', u'spoons', u'belong', u'too', u'liquor', u'bland', u'michael', u'pepper', u'spray', u'stout', u'remembers', u'ac', u'everything', u'furnace', u'installing', u'unit', u'acknowledged', u'courses', u'golf', u'tonight', u'brent', u'era', u'matt', u'\"just', u'arpaio', u'metro', u'gumbo', u'house', u'beignets', u'chipotle', u'family', u'polenta', u'antique', u'offers', u'frittatas', u'janet', u'perfect', u'sampling', u'trip', u'okra', u'saint', u'manhattan', u'gnocchi', u'alot', u'heritage', u'science', u'\"green\"', u'azcentral', u'development', u'white', u'towners', u'knocked', u'phx', u'uptown', u'vig', u'orgasm', u'cups', u'vera', u'lack', u'girls', u'az', u'mattress', u'glitter', u'furniture', u\"men's\", u'rocket', u'andy', u'museum', u'lamb', u'grilled', u'patty', u'focaccia', u'churros', u'swordfish', u'lights', u'gluten', u'funny', u'lol', u'railing', u'cafe', u\"tj's\", u'fuego', u'deals', u'bird', u'stations', u'browns', u'beaver', u'gravlax', u'defiantly', u'crackers', u'rap', u'reach', u'summit', u'fillet', u'repaired', u'windshield', u'anniversary', u'skate', u'veggie', u'locating', u'surpassed', u'persons', u'shoestring', u'suns', u'bike', u'photo', u'less', u'tanning', u'time', u'needy', u'eggrolls', u\"morton's\", u'n', u'pancit', u'clams', u'have', u'wreck', u'wicked', u'chart', u'egg', u'brulee', u'creme', u'bakery', u'leaf', u'loose', u'teas', u'brownies', u'holiday', u'lobbys', u'lisa', u'prosciutto', u'address', u'questions', u'spots', u'patisserie', u'breath', u'calamari', u'shannon', u'herself', u'scottsdale', u'copper', u'fuel', u'dangerous', u'tater', u'tots', u'route', u'oink', u'cook', u'dont', u'seasoned', u'yu', u'curries', u'steps', u'repeat', u'iphone', u'garner', u'sometimes', u'national', u'appetizers', u'girlfriends', u'wines', u'scandinavian', u'aloo', u'tikki', u'stadiums', u'vendors', u'clips', u'munchies', u'queso', u'tenders', u'wontons', u'curds', u'lodge', u'microwaved', u'craving', u'copy', u'original', u'biscuit', u'tooth', u'zoo', u'wiz', u'daiya', u'sorts', u'nation', u'distrito', u'salespeople', u'marie', u'pane', u'greyhound', u'racing', u'success', u'ceviche', u'no', u'pico', u'stale', u'ranch', u'fist', u'hawaiian', u'filibertos', u'haha', u'saffron', u'cash', u'toes', u'wave', u'octopus', u'sour', u'lanes', u'target', u'acai', u'birthdays', u'hubs', u'concrete', u'malt', u'mixer', u'poolside', u\"mastro's\", u'jesse', u'noir', u'surprise', u'trendy', u'bars', u'old', u'rooftop', u'town', u'colossal', u'prime', u'rib', u'\"i', u'fro', u'yo', u'veined', u'button', u'courtyard', u'bevmo', u\"lee's\", u'drinker', u'fashioned', u'parlor', u'exhibits', u'desk', u'messed', u'\"good', u'cares', u'filthy', u'brings', u'pb', u'cuz', u'jose', u'savory', u'boxty', u'expiration', u'vests', u'womens', u'kitchen', u'latest', u'laughing', u'boom', u'hagen', u'tells', u\"america's\", u'prices', u'mouthwatering', u'hawaii', u'stationed', u'danielle', u'girl', u\"mimi's\", u'smoothly', u'crepes', u'fi', u'jobot', u'wi', u'potatoes', u'bad', u'hay', u'tile', u'tub', u'vanity', u'club\"', u'wrapped', u'bitchy', u'outside', u'jersey', u'em', u'cole', u\"jake's\", u'recipt', u'utensils', u'blind', u'dad', u'entered', u'granola', u'fishing', u'lakes', u'sw', u'hubby', u'moved', u'onto', u'\"cha', u'japan', u'racks', u'scrub', u'gel', u'nail', u'blu', u'exotic', u'final', u'smart', u'departed', u'cleanest', u'unfriendly', u'machines', u'root', u'regularly', u'dental', u'dentist', u'record', u'doughnut', u'doughnuts', u'lectures', u'crepe', u'flying', u'grand', u'barely', u'snails', u'steamers', u'athletic', u'foot', u'locker', u'crave', u\"you'd\", u'preserve', u'pigs', u'collection', u'church', u'grace', u'hope', u'pastor', u'bun', u'court', u'fully', u'harder', u'taught', u'often', u'boxing', u'nourish', u'subway', u'strike', u'restroom', u'ace', u'shakes', u'sonic', u'under', u'muffins', u\"dave's\", u'enter', u'cafeteria', u'sides', u'bowl\"', u'bruce', u'convenient', u'cum', u'serve', u'shade', u'lamps', u'ugh', u'hand', u'moldy', u'exceeded', u'expectations', u'middle', u'unedible', u'info', u'margaritaville', u'meatloaf', u'tenderloin', u'barrel', u'cracker', u'bleu', u'nearby', u'west', u'managers', u'each', u'got', u'sharpie', u'dish', u'washed', u'drafts', u'meeting', u'paradise', u'travel', u'debit', u'grayhawk', u'bourbon', u'dishes', u'paintings', u'pours', u'tiny', u'bank', u'if', u'won', u'slides', u'appointment', u'toys', u'treats', u'waiter', u'silly', u'anthem', u'turkish', u'horchata', u'sara', u'v', u'unimpressed', u'sunnyslope', u\"dick's\", u'yeah', u'void', u'perplexed', u'double', u\"everyone's\", u'hooters', u'flute', u'suitable', u'walk', u'lawn', u'ultimate', u'=', u'looks', u'microwave', u're', u'hashbrowns', u'blowdry', u'keg', u'benedict', u'yolk', u'allow', u'manager', u'brews', u'eatery', u'rush', u'fairness', u'rootbeer', u'been', u'veteran', u'bit', u'volunteer', u'sorry', u'lattes', u'peppermint', u'patio', u'samples', u'zinc', u'camp', u'cashier', u'doggie', u'has', u'refreshed', u'train', u'plantain', u'bookstore', u'marcellino', u'feet', u'macarons', u'help', u'wives', u'computer', u'email', u'pc', u'decided', u'motel', u'celebrate', u'trying', u'quinoa', u'upper', u'chickens', u'camelview', u'screening', u'sakana', u'cabbage', u'gristle', u'haus', u'schnitzel', u'beep', u'brioni', u'thousand', u'branch', u'california', u'meetup', u'peppercorn', u'yes', u'incredible', u'locked', u'cajun', u'jennifer', u'orleans', u'po', u'gravel', u'materials', u'staff', u'luggage', u'taken', u'hyatt', u'customers', u'nike', u'cheesesteak', u'forefathers', u'peppers', u'dan', u'gabi', u'accompaniments', u'butterfly', u'exhibit', u'die', u'cutlet', u'farmers', u'dust', u'teeth', u'kristi', u'pate', u'similarly', u'dumplings', u'stands', u'field', u'outfield', u'pit', u'rubbery', u'vibe', u'ao', u'stuffed', u'gio', u'length', u'long', u'mover', u'reputation', u'scrambled', u'dobson', u'mural', u'boost', u'feel', u'stream', u'willow', u'flavour', u'peas', u'scallops', u'cowboys', u'flat', u'retailers', u'dosa', u\"bear's\", u'tasted', u'mazing', u'month', u'cooperstown', u'boyfriends', u'material', u'fly', u'spanish', u'mashed', u'membership', u'tasting', u'ohio', u'cause', u'jacket', u'brewpub', u'los', u'personable', u'ipod', u'purchasing', u'practice', u'bandera', u'cornbread', u'roasted', u'malbec', u'foie', u'gras', u'friendly', u'security', u'desserts', u'feature', u'fields', u'kona', u'lil', u'warm', u'ziti', u'building', u'monkey', u'sitting', u'sketchy', u'sombreros', u'ham', u'dunkin', u'waits', u'unique', u'busy', u'smell', u'sensor', u'fruity', u'pebbles', u'daycare', u'mega', u'carlos', u'ghost', u'obligatory', u'diner', u'veg', u'blt', u'knot', u'icky', u'carrot', u'visitors', u'events', u'frosting', u'short', u'dip', u'chuck', u'tejas', u'glass', u'martini', u'\"off\"', u'tinted', u'never', u\"else's\", u'chefs', u'preparing', u'adventure', u'oils', u'cashiers', u'result', u'choice', u'fruit', u'kiwi', u'mango', u'soggy', u'almond', u'croissants', u'plain', u'warrant', u'spicier', u'mon', u'night', u'whats', u'cafes', u'department', u'goes', u'quite', u'games', u'relaxed', u'compared', u'transit', u'barbecue', u'bobby', u'gallery', u'confusion', u'tryst', u\"karsh's\"]\n",
        "2549\n"
       ]
      }
     ],
     "prompt_number": 60
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print trnginp.shape\n",
      "\n",
      "\n",
      "from sklearn.linear_model import LinearRegression\n",
      "\n",
      "trnglabels = review_df['useful_votes'][:3000]\n",
      "LRModel = LinearRegression()\n",
      "LRModel.fit(trnginp,trnglabels)\n",
      "modelpar = LRModel.coef_\n",
      "print min(modelpar),max(modelpar), modelpar\n",
      "\n",
      "#tip = list()\n",
      "#testsample = tip.append(random)\n",
      "#count_vect.fit_transform(review_df['text'][100:105])\n",
      "#print LRModel.predict(random)\n",
      "\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(3000, 2549)\n",
        "-68.5216342876"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 73.768159965 [ 0.0056477   0.1363848   0.11416808 ..., -2.39524026 -4.0022259  -3.553558  ]\n"
       ]
      }
     ],
     "prompt_number": 53
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "len(count_vect.get_feature_names())\n",
      "testrange = count_vect.fit_transform(review_df['text'][3001:3022])\n",
      "\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 54
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#print Counter(trnginp.toarray()[0])\n",
      "pred_results = LRModel.predict(testrange.toarray())\n",
      "#print pred_results\n",
      "print zip(review_df['useful_votes'][3001:3022].values,pred_results)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[(3, 0.10722253192732678), (3, 5.1307727750280891), (2, 1.8497340814818808), (2, 0.86242083409759784), (12, -3.243073858639637), (3, 0.74795571782412962), (2, 11.572108743799092), (15, 16.557388030943269), (2, 9.1431829674520966), (4, 7.1834717182221199), (2, 8.7959294859872692), (2, 6.2363301565632714), (6, 16.773572365681332), (2, 1.8434483663144543), (6, 9.8385211685916811), (9, 6.2991945687837463), (4, 15.234917834189005), (2, 5.3461336062475908), (3, 3.0171191624784326), (2, 3.8597769858171067), (2, 12.20388887502394)]\n"
       ]
      }
     ],
     "prompt_number": 55
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Ideas:\n",
      "1. Find Best stop word method\n",
      "2. Take into account punctuation\n",
      "3. Take into account prices in reviews (eg $14.95 = $**.** for any number)\n",
      "4. n-gram v. bi-gram"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "index = Counter()\n",
      "prices = []\n",
      "\n",
      "#nltk.word_tokenize(raw)\n",
      "\n",
      "for row in islice(review_df.iterrows(),None):\n",
      "    for word in re.sub('\\W',\" \",row[1]['text']).split(' '):\n",
      "        if word != \"\" and word != \" \":\n",
      "            if word.lower() not in stops:\n",
      "                if '$' in word:\n",
      "                    prices.append(word)\n",
      "                else:\n",
      "                    index[word.lower()] += 1\n",
      "            "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "heading",
     "level": 4,
     "metadata": {},
     "source": [
      "tokenize any price using a $ (regex example)"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print prices[0:10]\n",
      "print len(prices)\n",
      "\n",
      "regs = ['$$' for p in prices if re.search('^\\$?', p)]\n",
      "print regs[0:10]\n",
      "print len(regs)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[]\n",
        "0\n",
        "[]\n",
        "0\n"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "heading",
     "level": 4,
     "metadata": {},
     "source": [
      "regex to find variations of the word 'love'"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "loves = [ word for word in index.keys() if re.search('^l+o+v+e+$', word) or re.search('^l+u+v+$',word) or re.search('^l+o+v+$',word) ]\n",
      "print loves[:]\n",
      "print len(loves)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "NameError",
       "evalue": "name 'index' is not defined",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
        "\u001b[1;32m<ipython-input-57-b997dfc55fa3>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mloves\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m \u001b[0mword\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mword\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mindex\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mre\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msearch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'^l+o+v+e+$'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mword\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mre\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msearch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'^l+u+v+$'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mword\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mre\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msearch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'^l+o+v+$'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mword\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;32mprint\u001b[0m \u001b[0mloves\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;32mprint\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mloves\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;31mNameError\u001b[0m: name 'index' is not defined"
       ]
      }
     ],
     "prompt_number": 57
    },
    {
     "cell_type": "heading",
     "level": 4,
     "metadata": {},
     "source": [
      "word_var(word) takes any word and makes the regex search pattern"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def word_var(word):\n",
      "    return str('^' + \"\".join([ l + \"+\" for l in word ]) + \"$\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pattern = word_var(\"awesome\")\n",
      "print pattern\n",
      "\n",
      "amazing = [ word for word in index.keys() if re.search(pattern,word) ]\n",
      "print amazing\n",
      "print len(amazing)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[u'awessome', u'aweesome', u'awesommmme', u'aawwwesooomme', u'aweeeeeeeeeeeeeesome', u'awesome', u'awesomee', u'awessomeeeee', u'awesomeeeeee', u'awesooooooome', u'awesooooome', u'aaawesome', u'awesoooommmee', u'aweeeeeesome']\n",
        "14\n"
       ]
      }
     ],
     "prompt_number": 18
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pattern = word_var(\"slow\")\n",
      "slow = [ word for word in index.keys() if re.search(pattern, word) ]\n",
      "print slow\n",
      "print len(slow)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[u'slooowww', u'slllloooww', u'ssllooww', u'slooooooooooow', u'slllllow', u'slllllllow', u'sssllllooowww', u'sloooowww', u'sllllloooowww', u'slooooowwwwwww', u'sloooow', u'sllloooooww', u'slowwww', u'sloooooooooowwwww', u'slllllooooowwwwww', u'sloowwwww', u'sllloooowwww', u'sllloooooow', u'slooooooww', u'slow', u'sllloooow', u'slooowwww', u'slooooow', u'slllloooooowwww', u'sloooowwww', u'slowwwwwwwwwwwwwwww', u'slowww', u'slllllllllllllllllllllllllloooooooooooooowwwwwww', u'slooow', u'sloooooooooooooooooooow', u'sloowww', u'ssssslllllooooooowwww', u'sloooooooooooowww', u'sloooooow', u'sloooowwwww', u'slowwwww', u'sloooooooooow', u'slooooooooow', u'slowwwwww', u'slllllloooooooooooooowwww', u'slooooooooooooow', u'slooooowwwww', u'slooooooow', u'sloooooooow']\n",
        "44\n"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "multiples = []\n",
      "word_corpus = index.keys()\n",
      "word_corpus.sort()\n",
      "word_corpus.sort(key=len, reverse=False) # http://stackoverflow.com/questions/4659524/how-to-sort-by-length-of-string-followed-by-alphabetical-order\n",
      "print word_corpus[:10]\n",
      "\n",
      "for check_word in islice(word_corpus,100):\n",
      "    pattern = word_var(check_word)\n",
      "    #print \"check\",check_word\n",
      "    variations = [ word for word in index.keys() if re.search(pattern, word ) ]\n",
      "    if len(variations) > 2:\n",
      "        print variations\n",
      "        multiples.append(variations)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[u'0', u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9']\n",
        "[u'00', u'000', u'0']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'111', u'11', u'1']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'2', u'22', u'222']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'3333', u'333', u'3', u'33']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'444', u'44', u'4']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'555', u'55', u'5']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'66', u'6666', u'666666666666666666666666666666666666', u'6']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'777', u'7', u'77']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'88', u'8888', u'8']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'9', u'9999', u'999999', u'99', u'999']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'____________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________', u'__', u'___', u'__________', u'____________________________________________________', u'_____________', u'__________________', u'_________________________________________________________', u'_________________________________________________', u'______________________', u'_________', u'____________________', u'__________________________', u'_______', u'______________________________________________', u'________________________________________', u'_________________', u'_____', u'___________________________________________', u'______________________________________________________', u'___________________', u'_______________________________________________________', u'________________________________________________________', u'_______________', u'____', u'___________________________________________________', u'_', u'__________________________________________________', u'_____________________________________________________________', u'________', u'______', u'_________________________________', u'_______________________________', u'______________', u'________________________']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'b', u'bb', u'bbb']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'cc', u'cccc', u'c', u'ccc']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'd', u'dd', u'ddd']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'eeeeeeeee', u'ee', u'eeee', u'eee', u'eeeeee', u'e']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'ffff', u'fff', u'ff', u'f']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'gggggg', u'g', u'gg', u'ggggg']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'h', u'hh', u'hhh']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'ii', u'iii', u'i', u'iiiii']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'jjj', u'jj', u'j']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'kk', u'k', u'kkk']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'll', u'lll', u'l']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'mmmmmmmmmmmmmmm', u'mmmmmmmmmmmmmmmmmm', u'mmmmmmmmm', u'mmmmmmmmmmmmm', u'mmmmmmm', u'mm', u'mmmmmmmm', u'mmmmmmmmmmmmmmmmm', u'mmmmmmmmmmmmmmmmmmmm', u'mmmmm', u'mmmmmmmmmmmmmmmmmmmmmmmmmmm', u'mmm', u'mmmmmmmmmmmmmm', u'mmmmmm', u'mmmmmmmmmmmmmmmm', u'mmmmmmmmmmm', u'mmmm', u'mmmmmmmmmmmmmmmmmmmmmmmm', u'm', u'mmmmmmmmmmmm', u'mmmmmmmmmmmmmmmmmmm', u'mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm', u'mmmmmmmmmm']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'ooooooo', u'ooooooooo', u'oo', u'ooooo', u'o', u'ooooooooooooo', u'oooooo', u'ooooooooooo', u'oooo', u'ooo', u'oooooooo']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'ppp', u'pp', u'p']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'qq', u'qqqqq', u'q']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'rrrr', u'r', u'rrrrr', u'rrrrrrrrrrrr', u'rr', u'rrrrrrr', u'rrr']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'ss', u'sss', u's', u'sssssss']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'tt', u'ttt', u't']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'u', u'uuuuu', u'uuuuuuu']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'xxxxxxxx', u'x', u'xxx', u'xxxx', u'xxxxx', u'xx']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'zzzzzzzzzzz', u'zzzz', u'zzzzzzzzzzzzzz', u'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz', u'zzzzzzzzzzzzzzzzz', u'zzzzzzzz', u'zzz', u'z', u'zzzzzz', u'zzzzzzz', u'zz', u'zzzzzzzzz', u'zzzzzzzzzz', u'zzzzz', u'zzzzzzzzzzzzz']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'00000001', u'01', u'001']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'05', u'005', u'0005']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'11000000', u'10000', u'100', u'1100', u'110', u'10000000', u'1000000', u'1000000000000000000000', u'1000', u'10']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'112', u'122', u'12']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'1113', u'113', u'13', u'133']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'144', u'14', u'114']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'1155', u'1115', u'115', u'155', u'15']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'1666666', u'1116', u'116', u'16', u'166']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'177', u'17', u'117']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'118', u'1888', u'18', u'188']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'199', u'1999', u'1199', u'119', u'19']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'1m', u'1mm', u'11m']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'200', u'2200', u'20', u'220', u'2000', u'2220']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'21', u'221', u'211']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'24', u'224', u'244']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'25', u'225', u'255']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'266', u'26', u'226']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'277', u'27', u'227']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'228', u'288', u'28']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[u'29', u'299', u'229', u'2229']"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 4,
     "metadata": {},
     "source": [
      "nltk"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#nltk.word_tokenize(raw)\n",
      "porter = nltk.PorterStemmer()\n",
      "\n",
      "\n",
      "tokens = []\n",
      "\n",
      "for row in islice(review_df.iterrows(),1000):\n",
      "    line = re.sub('\\n',\"\",row[1]['text']).split(' ')\n",
      "    tokens.extend(line)\n",
      "    \n",
      "stems = [ porter.stem(t) for t in tokens ]\n",
      "print len(set(tokens))\n",
      "print len(set(stems))\n",
      "\n",
      "print stems[0:100]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "19735\n",
        "17551\n",
        "[u'My', u'wife', u'took', u'me', u'here', u'on', u'my', u'birthday', u'for', u'breakfast', u'and', u'it', u'wa', u'excellent.', '', u'The', u'weather', u'wa', u'perfect', u'which', u'made', u'sit', u'outsid', u'overlook', u'their', u'ground', u'an', u'absolut', u'pleasure.', '', u'Our', u'waitress', u'wa', u'excel', u'and', u'our', u'food', u'arriv', u'quickli', u'on', u'the', u'semi-busi', u'Saturday', u'morning.', '', u'It', u'look', u'like', u'the', u'place', u'fill', u'up', u'pretti', u'quickli', u'so', u'the', u'earlier', u'you', u'get', u'here', u'the', u'better.Do', u'yourself', u'a', u'favor', u'and', u'get', u'their', u'Bloodi', u'Mary.', '', u'It', u'wa', u'phenomen', u'and', u'simpli', u'the', u'best', u\"I'v\", u'ever', u'had.', '', u\"I'm\", u'pretti', u'sure', u'they', u'onli', u'use', u'ingredi', u'from', u'their', u'garden', u'and', u'blend', u'them', u'fresh', u'when', u'you', u'order', u'it.']\n"
       ]
      }
     ],
     "prompt_number": 48
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Create Business DF"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "review_json = open(yelp_dir + \"/\" + data_files[0])\n",
      "business_for_df = [ json.loads(line) for line in islice(review_json,None) ]\n",
      "business_df = pd.DataFrame(business_for_df)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "business_df[:1]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 4,
     "metadata": {},
     "source": [
      "How many categories are there?"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "\n",
      "cats = []\n",
      "for row in islice(business_df.iterrows(),10):\n",
      "    print row[1]['categories']\n",
      "    cats.extend(row[1]['categories'])\n",
      "    \n",
      "print len(set(cats))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Try scikit-learn classifier"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# 229907\n",
      "\n",
      "train = {}\n",
      "\n",
      "train['data'] = []\n",
      "train['target'] = []\n",
      "train['target_names'] = ['useful','not']\n",
      "\n",
      "for row in islice(review_df.iterrows(),0,200000):\n",
      "    #print row[1]['text']\n",
      "    #print row[1]['votes']['useful']\n",
      "    \n",
      "    train['data'].append(row[1]['text'])\n",
      "    if row[1]['votes']['useful'] > 4:\n",
      "        train['target'].append([0])\n",
      "    else:\n",
      "        train['target'].append([1])\n",
      "    \n",
      "test = {}    \n",
      "\n",
      "test['data'] = []\n",
      "test['target'] = []\n",
      "test['target_names'] = ['useful','not']\n",
      "\n",
      "for row in islice(review_df.iterrows(),200001,229907):\n",
      "    #print row[1]['text']\n",
      "    #print row[1]['votes']['useful']\n",
      "    \n",
      "    train['data'].append(row[1]['text'])\n",
      "    if row[1]['votes']['useful'] > 4:\n",
      "        train['target'].append([0])\n",
      "    else:\n",
      "        train['target'].append([1])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 31
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.feature_extraction.text import CountVectorizer\n",
      "count_vect = CountVectorizer()\n",
      "X_train_counts = count_vect.fit_transform(train['data'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "KeyboardInterrupt",
       "evalue": "",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
        "\u001b[0;32m<ipython-input-32-85ec01f40558>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeature_extraction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mCountVectorizer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mcount_vect\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCountVectorizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mX_train_counts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcount_vect\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'data'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
        "\u001b[0;32m/Library/Python/2.7/site-packages/sklearn/feature_extraction/text.pyc\u001b[0m in \u001b[0;36mfit_transform\u001b[0;34m(self, raw_documents, y)\u001b[0m\n\u001b[1;32m    746\u001b[0m         \u001b[0;31m# we really sure want we want to drop them? They take some memory but\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    747\u001b[0m         \u001b[0;31m# can be useful for corpus introspection\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 748\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_term_count_dicts_to_matrix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mterm_counts_per_doc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    749\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    750\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mraw_documents\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m/Library/Python/2.7/site-packages/sklearn/feature_extraction/text.pyc\u001b[0m in \u001b[0;36m_term_count_dicts_to_matrix\u001b[0;34m(self, term_count_dicts)\u001b[0m\n\u001b[1;32m    621\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mterm_count_dict\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mterm_count_dicts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    622\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mterm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcount\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mterm_count_dict\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miteritems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 623\u001b[0;31m                 \u001b[0mj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvocabulary\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mterm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    624\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mj\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    625\u001b[0m                     \u001b[0mi_indices\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
       ]
      }
     ],
     "prompt_number": 32
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.feature_extraction.text import TfidfTransformer\n",
      "X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)\n",
      "X_train_tfidf.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}