twitter-searcher-v2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import twitter\n",
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "#import matplotlib.pyplot as plt\n",
    "\n",
    "# Import unquote to prevent url encoding errors in next_results\n",
    "# The urllib module is split into urllib.parse, urllib.request, \n",
    "# and urllib.error in Python 3\n",
    "# If running this in Python 2, change urllib.parse to urllib\n",
    "from urllib import unquote\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# OAuth login function for instantiating Twitter API object\n",
    "def oauth_login():\n",
    "    # XXX: Go to http://twitter.com/apps/new to create an app and get values\n",
    "    # for these credentials that you'll need to provide in place of these\n",
    "    # empty string values that are defined as placeholders.\n",
    "    # See https://dev.twitter.com/docs/auth/oauth for more information \n",
    "    # on Twitter's OAuth implementation.\n",
    "    \n",
    "\n",
    "    \n",
    "    auth = twitter.oauth.OAuth(OAUTH_TOKEN, OAUTH_TOKEN_SECRET,\n",
    "                               CONSUMER_KEY, CONSUMER_SECRET)\n",
    "    \n",
    "    twitter_api = twitter.Twitter(auth=auth)\n",
    "    return twitter_api"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Find the up to 600 most recent tweets containing a given hashtag\n",
    "# The returned object contains a massive amount of data\n",
    "def find_tweets(hashtag):\n",
    "    # Ensure that the hashtag argument is a string\n",
    "    assert type(hashtag) == str, 'Argument hashtag must be a string'\n",
    "    # Ensure that the hashtag argument is, indeed, a hashtag \n",
    "    # in quotes, i.e. a string beginning with a pound sign\n",
    "    assert hashtag[0] == '#', 'hashtag string must begin with a #'\n",
    "    # Instantiate the Twitter API object \n",
    "    twitter_api = oauth_login()\n",
    "    # Begin searching the Twitter API for tweets containing hashtag\n",
    "    # Twitter only lets you find 100 tweets at a time \n",
    "    # (by default, the 100 most recent)\n",
    "    search_results = twitter_api.search.tweets(q=hashtag, count=100, result_type='recent')\n",
    "    \n",
    "    # Extract the information on the (up to) 100 most recent tweets\n",
    "    # as a list\n",
    "    statuses = search_results['statuses']\n",
    "    \n",
    "    # Iterate through 5 more batches of results by following the cursor \n",
    "    # back in time\n",
    "    for _ in range(5):\n",
    "        #print(\"Length of statuses\", len(statuses))\n",
    "        try:\n",
    "            next_results = search_results['search_metadata']['next_results']\n",
    "        # The as statement is required in Python 3  \n",
    "        # A comma would be required instead for Python 2.5 and earlier  \n",
    "        # Python 2.6 and 2.7 support both the comma and the as statement\n",
    "        except KeyError as e: # No more results when next_results doesn't exist\n",
    "            break\n",
    "        \n",
    "        # Create a dictionary from next_results, which has the following form:\n",
    "        # ?max_id=313519052523986943&q=NCAA&include_entities=1\n",
    "        kwargs = dict([kv.split('=') for kv in unquote(next_results[1:]).split(\"&\") ])    \n",
    "        \n",
    "        # Search for the 100 next most recent tweets\n",
    "        search_results = twitter_api.search.tweets(**kwargs)\n",
    "        # Append the results of the last search to the statuses list\n",
    "        statuses += search_results['statuses']\n",
    "        \n",
    "    return statuses\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Resample/re-bin the time stamps to get a count of the tweets\n",
    "# by time interval, which will vary by the the difference between\n",
    "# the first and last time stamps\n",
    "def resample_time_stamps(time_df):\n",
    "    # Ensure that time_df is a DataFrame\n",
    "    # The assert statement looks like this because \n",
    "    # we imported pandas as pd\n",
    "    assert type(time_df) == pd.core.frame.DataFrame, 'Argument must be a pandas DataFrame'\n",
    "    \n",
    "    # Determine the range of times\n",
    "    time_index  = time_df.index\n",
    "    diff_year   = abs(time_index[0].year   - time_index[-1].year)\n",
    "    diff_month  = abs(time_index[0].month  - time_index[-1].month)\n",
    "    diff_day    = abs(time_index[0].day    - time_index[-1].day)\n",
    "    diff_hour   = abs(time_index[0].hour   - time_index[-1].hour)\n",
    "    diff_minute = abs(time_index[0].minute - time_index[-1].minute)\n",
    "    diff_second = abs(time_index[0].second - time_index[-1].second)\n",
    "    # Use the range of times to resample the data, \n",
    "    # and thus determine the plot format\n",
    "    # Perhaps adjust this later\n",
    "    if diff_year > 0:\n",
    "        # Resample/bucket by month\n",
    "        # The fillna(0) command fills any bins with NaN in them with 0s\n",
    "        time_df = time_df.resample('M', how='sum').fillna(0)\n",
    "        bin_width = 'month'\n",
    "    elif diff_month > 0:\n",
    "        if diff_month > 4:\n",
    "            # Resample by week\n",
    "            time_df = time_df.resample('W', how='sum').fillna(0)\n",
    "            bin_width = 'week'\n",
    "        else: \n",
    "            # Resample by day\n",
    "            time_df = time_df.resample('D', how='sum').fillna(0)\n",
    "            bin_width = 'day'\n",
    "    elif diff_day > 0:\n",
    "        if diff_day <= 10:\n",
    "            # Resample by hour\n",
    "            time_df = time_df.resample('H', how='sum').fillna(0)\n",
    "            bin_width = 'hour'\n",
    "        elif diff_day <= 21:\n",
    "            # Resample by 6 hours\n",
    "            time_df = time_df.resample('6H', how='sum').fillna(0)\n",
    "            bin_width = '6 hours'\n",
    "        else: \n",
    "            # Resample by day\n",
    "            time_df = time_df.resample('D', how='sum').fillna(0)\n",
    "            bin_width = 'day'\n",
    "    elif diff_hour > 0:\n",
    "        if diff_hour <= 3:\n",
    "            # Resample by minute\n",
    "            time_df = time_df.resample('T', how='sum').fillna(0)\n",
    "            bin_width = 'minute'\n",
    "        elif diff_hour <= 12:\n",
    "            # Resample by 5 minutes\n",
    "            time_df = time_df.resample('5T', how='sum').fillna(0)\n",
    "            bin_width = '5 minutes'\n",
    "        else: \n",
    "            # Resample by 10 minutes\n",
    "            time_df = time_df.resample('10T', how='sum').fillna(0)\n",
    "            bin_width = '10 minutes'\n",
    "    elif diff_minute > 0:\n",
    "        if diff_minute <= 3:\n",
    "            # Resample by second\n",
    "            time_df = time_df.resample('S', how='sum').fillna(0)\n",
    "            bin_width = 'second'\n",
    "        elif diff_minute <= 12:\n",
    "            # Resample by 5 seconds\n",
    "            time_df = time_df.resample('5S', how='sum').fillna(0)\n",
    "            bin_width = '5 seconds'\n",
    "        else: \n",
    "            # Resample by 10 seconds\n",
    "            time_df = time_df.resample('10S', how='sum').fillna(0)\n",
    "            bin_width = '10 seconds'\n",
    "    elif abs(time_index[0].second - time_index[-1].second) > 0:\n",
    "        # Resample/bucket (reorder) by second\n",
    "        time_df = time_df.resample('S', how='sum').fillna(0)\n",
    "        bin_width = 'second'\n",
    "    else: \n",
    "        # Resample/bucket (reorder) by second\n",
    "        time_df = time_df.resample('S', how='sum').fillna(0)\n",
    "        bin_width = 'second'\n",
    "    \n",
    "    return time_df, bin_width"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Plot the time distribution of the (up to) 600 \n",
    "# most recent tweets containing the desired hashtag\n",
    "# Also returns a DataFrame containing the times \n",
    "# and locations (if available) of the tweets\n",
    "def map_tweets(hashtag):\n",
    "    # Ensure that the hashtag argument is a string\n",
    "    assert type(hashtag) == str, 'Argument hashtag must be a string'\n",
    "    # Ensure that the hashtag argument is, indeed, a hashtag \n",
    "    # in quotes, i.e. a string beginning with a pound sign\n",
    "    assert hashtag[0] == '#', 'hashtag string must begin with a #'\n",
    "    \n",
    "    # Get all the data on the most recent tweets\n",
    "    statuses = find_tweets(hashtag)\n",
    "    \n",
    "    # Initialize and populate a NumPy array with \n",
    "    # the time stamps of the tweets contained in \n",
    "    # statuses, in reverse chronological order\n",
    "    time_stamps = np.array([])\n",
    "    for i in range(len(statuses)):\n",
    "        time_stamps = np.append(time_stamps, statuses[i]['created_at'])\n",
    "    \n",
    "    # Initialize and populate a NumPy array with \n",
    "    # the coordinates of the tweets contained in \n",
    "    # statuses, if available\n",
    "    coordinates = np.array([])\n",
    "    for i in range(len(statuses)):\n",
    "        coordinates = np.append(coordinates, statuses[i]['coordinates'])\n",
    "    \n",
    "    # Create an array of ones for every tweet\n",
    "    ones_array = np.ones(len(time_stamps))\n",
    "    # Create a Pandas DatetimeIndex of the time stamps\n",
    "    time_index = pd.DatetimeIndex(time_stamps)\n",
    "    # Create a Pandas DataFrame showing one tweet \n",
    "    # for each time stamp\n",
    "    time_df = pd.DataFrame(ones_array, index=time_index, columns=['Tweets'])\n",
    "    # Create a DataFrame associating a location with \n",
    "    # each time stamp, if available\n",
    "    coord_df = pd.DataFrame(coordinates, index=time_index, columns=['Coordinates'])\n",
    "    \n",
    "    # Resample the DataFrame time_df\n",
    "    time_df, bin_width = resample_time_stamps(time_df)\n",
    "    \n",
    "    # Create the plot\n",
    "    time_plot = time_df.plot(legend=False)\n",
    "    time_plot.set_xlabel('Date/Time')\n",
    "    time_plot.set_ylabel('Tweets per '+bin_width)\n",
    "    time_plot.set_title('Frequency of tweets containing hashtag '+hashtag)\n",
    "    \n",
    "    return time_plot, coord_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Find as many tweets as possible containing a given hashtag \n",
    "# known to originate within search_radius of coords\n",
    "# DO NOT CALL THIS UNLESS A TWITTER API OBJECT HAS BEEN INSTANTIATED!\n",
    "def find_all_tweets(hashtag, coords, search_radius, api_calls):\n",
    "    # First, ensure that the arguments are all of the \n",
    "    # proper type\n",
    "    assert type(hashtag)       == str, 'Argument hashtag must be a string'\n",
    "    assert type(coords)        == str, 'Argument coords must be a string'\n",
    "    assert type(search_radius) == str, 'Argument search_radius must be a string'\n",
    "    assert type(api_calls)     == int, 'Argument api_calls must be an integer'\n",
    "    # Ensure that the hashtag argument is, indeed, a hashtag \n",
    "    # in quotes, i.e. a string beginning with a pound sign\n",
    "    #assert hashtag[0] == '#', 'hashtag string must begin with a #'\n",
    "    # Ensure that the search radius is expressed in either mi or km\n",
    "    assert (search_radius[-2:] == 'mi') or (search_radius[-2:] == 'km'), \\\n",
    "      'search_radius must end in mi or km'\n",
    "    \n",
    "    # Twitter limits us to 180 calls to its API within a 15-minute span \n",
    "    # Setting this waiting period (in seconds) will allow us to pause \n",
    "    # the evaluation of this loop as necessary without having to restart \n",
    "    # it manually\n",
    "    waiting_period = 15*60 \n",
    "    # I know fully well that this works out to 900 seconds \n",
    "    # I merely thought that expressing the waiting period in min*60 sec/min \n",
    "    # would make it more obvious how long the waiting period is\n",
    "    \n",
    "    # Check to make sure we're not about to hit the API rate limit\n",
    "    # If we are, wait 15 minutes before continuing \n",
    "    # and reset the API call counter\n",
    "    if api_calls == 180:\n",
    "        time.sleep(waiting_period)\n",
    "        api_calls = 0\n",
    "    \n",
    "    # Find the 100 most recent tweets containing hashtag \n",
    "    # which originated within search_radius of coords, \n",
    "    # then extract the relevant information\n",
    "    geocode_str = coords + ',' + search_radius\n",
    "    search_results = \\\n",
    "      twitter_api.search.tweets(q=hashtag,count=100,result_type='recent',\n",
    "                                geocode=geocode_str)\n",
    "    statuses = search_results['statuses']\n",
    "    # Update the counter\n",
    "    api_calls += 1\n",
    "    \n",
    "    # Iterate through as many batches of results as possible \n",
    "    # by following the cursor back in time\n",
    "    # First, initialize the Boolean variable indicating whether \n",
    "    # more search results exist\n",
    "    more_results = True\n",
    "    while more_results:\n",
    "        #print(\"Length of statuses\", len(statuses))\n",
    "        try:\n",
    "            next_results = search_results['search_metadata']['next_results']\n",
    "        # The as statement is required in Python 3  \n",
    "        # A comma would be required instead for Python 2.5 and earlier  \n",
    "        # Python 2.6 and 2.7 support both the comma and the as statement\n",
    "        except KeyError as e: # No more results when next_results doesn't exist\n",
    "            more_results = False\n",
    "            break\n",
    "        \n",
    "        # Create a dictionary from next_results, which has the following form:\n",
    "        # ?max_id=313519052523986943&q=NCAA&include_entities=1\n",
    "        kwargs = dict([kv.split('=') for kv in unquote(next_results[1:]).split(\"&\") ]) \n",
    "        \n",
    "        # Check to make sure we're not about to hit the API rate limit\n",
    "        # If we are, wait 15 minutes before continuing\n",
    "        # and reset the API call counter\n",
    "        if api_calls == 180:\n",
    "            time.sleep(waiting_period)\n",
    "            api_calls = 0\n",
    "        # Search for the 100 next most recent tweets\n",
    "        search_results = twitter_api.search.tweets(**kwargs)\n",
    "        # Append the results of the last search to the statuses list\n",
    "        statuses += search_results['statuses']\n",
    "        # Update the counter\n",
    "        api_calls += 1\n",
    "    \n",
    "    return statuses, api_calls\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Extract the time stamp, coordinates, and user location \n",
    "# from the tweets, if available\n",
    "def parse_tweets_for_coordinate(statuses, coord,city):\n",
    "    # Initialize a list of the time stamps of the tweets \n",
    "    # contained in statuses, in reverse chronological order\n",
    "    \n",
    "    # Initialize a list of user locations\n",
    "    user_locations = [] \n",
    "    locationInfo = [];\n",
    "    for i in range(len(statuses)): \n",
    "        element = {};\n",
    "        element['type'] = 'Feature';\n",
    "        element['properties'] ={'timestamp':statuses[i]['created_at']}; \n",
    "        geomElement = {};\n",
    "        geomElement['type']= \"Point\";\n",
    "        a = coord.split(',');\n",
    "        c2 = [float(a[1]), float(a[0])]\n",
    "        geomElement['coordinates']=c2;\n",
    "        element['geometry'] = geomElement;\n",
    "        element['city'] = city;\n",
    "        locationInfo.append(element);\n",
    "         \n",
    "    #     # Create an array of ones for every tweet\n",
    "    #     ones_array = np.ones(len(time_stamps))\n",
    "    #     # Create a Pandas DatetimeIndex of the time stamps\n",
    "    #     time_index = pd.DatetimeIndex(time_stamps)\n",
    "    #     # Create a Pandas DataFrame showing one tweet \n",
    "    #     # for each time stamp\n",
    "    #     time_df = pd.DataFrame(ones_array, index=time_index, columns=['Tweets'])\n",
    "    #     # Create a DataFrame associating a GPS coordinate and user location  \n",
    "    #     # with each time stamp, if available\n",
    "    #     coord_loc_data = {'Coordinates': tweet_coords, 'User Location': user_locations}\n",
    "    #     coord_loc_df = pd.DataFrame(coord_loc_data, index=time_index)\n",
    "    #     # Filter out entries without coordinates\n",
    "    #     coord_loc_df = coord_loc_df[(coord_loc_df['Coordinates'] != 'None')]\n",
    "    \n",
    "    return locationInfo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "TwitterHTTPError",
     "evalue": "Twitter sent status 429 for URL: 1.1/search/tweets.json using parameters: (count=100&geocode=40.6643%2C-73.9385%2C30km&oauth_consumer_key=JrPeKVNTJI8aK4dPQ4cI1qgJb&oauth_nonce=16327731599181864152&oauth_signature_method=HMAC-SHA1&oauth_timestamp=1438381894&oauth_token=250500792-NtyWwlGsZq54lMqZpiSBG70tUAOr1fKdbJdz4y5L&oauth_version=1.0&q=HillaryClinton&result_type=recent&oauth_signature=Xa9Xjqw6itqbE%2F4ZMxywtm5hxE4%3D)\ndetails: {u'errors': [{u'message': u'Rate limit exceeded', u'code': 88}]}",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTwitterHTTPError\u001b[0m                          Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-45-4fe1d95875e1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     41\u001b[0m         \u001b[0;31m# Find as many tweets as possible containing hashtag\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m         \u001b[0;31m# originating within search_radius of coords\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 43\u001b[0;31m         \u001b[0mstatuses\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mapi_calls\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfind_all_tweets\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhashtag\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcoords\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msearch_radius\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mapi_calls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     44\u001b[0m         \u001b[0;32mprint\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstatuses\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-33-8dcf9bf3b1e8>\u001b[0m in \u001b[0;36mfind_all_tweets\u001b[0;34m(hashtag, coords, search_radius, api_calls)\u001b[0m\n\u001b[1;32m     36\u001b[0m     \u001b[0mgeocode_str\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcoords\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m','\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msearch_radius\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m     search_results =       twitter_api.search.tweets(q=hashtag,count=100,result_type='recent',\n\u001b[0;32m---> 38\u001b[0;31m                                 geocode=geocode_str)\n\u001b[0m\u001b[1;32m     39\u001b[0m     \u001b[0mstatuses\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msearch_results\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'statuses'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m     \u001b[0;31m# Update the counter\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m//anaconda/lib/python2.7/site-packages/twitter/api.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m    306\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle_response_with_retry\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muri\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_timeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    307\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 308\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle_response\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muri\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_timeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    309\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    310\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_handle_response\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muri\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_timeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m//anaconda/lib/python2.7/site-packages/twitter/api.pyc\u001b[0m in \u001b[0;36m_handle_response\u001b[0;34m(self, req, uri, arg_data, _timeout)\u001b[0m\n\u001b[1;32m    339\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    340\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 341\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mTwitterHTTPError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muri\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    343\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_handle_response_with_retry\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muri\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_timeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTwitterHTTPError\u001b[0m: Twitter sent status 429 for URL: 1.1/search/tweets.json using parameters: (count=100&geocode=40.6643%2C-73.9385%2C30km&oauth_consumer_key=JrPeKVNTJI8aK4dPQ4cI1qgJb&oauth_nonce=16327731599181864152&oauth_signature_method=HMAC-SHA1&oauth_timestamp=1438381894&oauth_token=250500792-NtyWwlGsZq54lMqZpiSBG70tUAOr1fKdbJdz4y5L&oauth_version=1.0&q=HillaryClinton&result_type=recent&oauth_signature=Xa9Xjqw6itqbE%2F4ZMxywtm5hxE4%3D)\ndetails: {u'errors': [{u'message': u'Rate limit exceeded', u'code': 88}]}"
     ]
    }
   ],
   "source": [
    "# Create giant libraries of tweet information \n",
    "# containing time stamp, geocode, and user location\n",
    "# Loop over hashtag, US metro area, and time, \n",
    "# as far back as the API will go\n",
    "\n",
    "#hashtags = ['#Hillary2016', '#Sanders2016', 'Trump2016']\n",
    "hashtags = ['HillaryClinton'];#, '#Sanders2016', 'Trump2016']\n",
    "cities = ['New York', 'Los Angeles', 'Chicago', \n",
    "          'Houston', 'Philadelphia', 'Phoenix', \n",
    "          'San Antonio', 'San Diego', 'Dallas', \n",
    "          'San Jose', 'Austin', 'Jacksonville', \n",
    "          'San Francisco', 'Indianapolis', 'Columbus', \n",
    "          'Fort Worth', 'Charlotte', 'Detroit', \n",
    "          'El Paso', 'Seattle', 'Denver', \n",
    "          'Washington DC']\n",
    "# Formatting each coordinate string as 'latitude, longitude' rather than\n",
    "# 'latitude,longitude' apparently plays havoc with the search.tweets() \n",
    "# function\n",
    "# Likewise, we apparently need geocode='latitude,longitude,search_radius' \n",
    "# in the argument list of search.tweets(), i.e. there can be no spaces \n",
    "# after the commas\n",
    "coordinates = ['40.6643,-73.9385', '34.046281, -118.231898',#replacing LA'34.0194,-118.4108', \n",
    "               '41.8376,-87.6818', \n",
    "               '29.7805,-95.3863', '40.0094,-75.1333', '33.5722,-112.088', \n",
    "               '29.4724,-98.5251', '32.8153,-117.135', '32.7757,-96.7967', \n",
    "               '37.2969,-121.8193', '30.3072,-97.756', '30.337,-81.6613', \n",
    "               '37.7751,-122.4193', '39.7767,-86.1459', '39.9848,-82.985', \n",
    "               '37.7821120598956,-122.400612831116', '35.2087,-80.8307', '42.383,-83.1022', \n",
    "               '31.8484,-106.427', '47.6205,-122.3509', '39.7618,-104.8806', \n",
    "               '38.9041,-77.0171']\n",
    "search_radius = '30km'\n",
    "\n",
    "# Initialize the API call counter\n",
    "api_calls = 0\n",
    "\n",
    "# Instantiate the Twitter API object\n",
    "twitter_api = oauth_login()\n",
    "for hashtag in hashtags:\n",
    "    locationInfo = [];\n",
    "    for city, coords in zip(cities, coordinates):\n",
    "        # Find as many tweets as possible containing hashtag \n",
    "        # originating within search_radius of coords\n",
    "        statuses, api_calls = find_all_tweets(hashtag, coords, search_radius, api_calls)\n",
    "        print len(statuses)\n",
    "        \n",
    "        # Extract the time stamps, coordinates, and user locations \n",
    "        # from the tweets found above\n",
    "        locationInfo += parse_tweets_for_coordinate(statuses,coords, city)\n",
    "    #GeoJSON Formatting of the features \n",
    "    geoj={};\n",
    "    geoj['type']=\"FeatureCollection\";\n",
    "    geoj['features']=locationInfo;\n",
    "    outputFile = 'friday_'+hashtag+'.geojson';\n",
    "    with open(outputFile, 'w') as outfile:\n",
    "        json.dump(geoj, outfile)\n",
    "            \n",
    "        print json.dumps(geoj, indent=1);        \n",
    "        \n",
    "        #         # Resample the DataFrame time_df\n",
    "        #         time_df, bin_width = resample_time_stamps(time_df)\n",
    "\n",
    "        #         # Create the plot\n",
    "        #         time_plot = time_df.plot(legend=False)\n",
    "        #         time_plot.set_xlabel('Date/Time')\n",
    "        #         time_plot.set_ylabel('Tweets per '+bin_width)\n",
    "        #         time_plot.set_title('Frequency of tweets containing hashtag '+hashtag\n",
    "        #                             +'\\n originating within '+search_radius\n",
    "        #                             +' of '+city)\n",
    "        \n",
    "        # Save the plot\n",
    "        #time_fig = time_plot.get_figure()\n",
    "        #fig_name = hashtag + '_' + city + '_' + search_radius + '_time_plot.png'\n",
    "        #time_fig.save(fig_name)\n",
    "        \n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['40.6643', '-73.9385']\n",
      "[40.6643, -73.9385]\n"
     ]
    }
   ],
   "source": [
    "c ='40.6643,-73.9385';\n",
    "\n",
    "a = c.split(',');\n",
    "print a\n",
    "c2 = [float(a[0]), float(a[1])]\n",
    "print c2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}