Completed fetch for tweets, added rake task to run. Also added rake t…

…ask to run all fetches.

app/models/tweet.rb

@@ -1,4 +1,4 @@
-require "rexml/document"
+require "json"
 require "open-uri"
 require "cgi"
 
@@ -14,87 +14,15 @@ def self.retrieve(event)
       since_id = latest_tweet_in_db.twitter_identifier
     end
 
-    endpoint = "http://search.twitter.com/search.atom"
+    endpoint = "http://search.twitter.com/search.json"
     call_uri = "#{endpoint}?q=#{CGI.escape("#" + tag)}&since_id=#{since_id}"
-    response = REXML::Document.new(open(call_uri).read)
-
-    puts response
-
-    REXML::XPath.match(response, "//photo").collect do |tweetelem|
-
+    response = JSON.parse(open(call_uri).read)
+
+    response["results"].each do |result|
+      tweet = Tweet.new(:body=> result["text"], :username => result["from_user"],
+        :twitter_identifier => result["id"].to_i, :posted_at => Time.parse(result["created_at"]),
+        :event_id => event.id)
+      tweet.save!
     end
   end
-
-
-    # 
-    # ## this method gets info about a certain photo
-    # def self.posted_date_for_image(id)
-    #   puts "checking posted date for image"
-    #   response = request("flickr.photos.getInfo", "photo_id" => id.to_s)
-    #   # why can't I make //photo/dates@posted work?
-    #   puts Time.at(REXML::XPath.match(response, "//photo/dates@posted")[0].attributes["posted"].to_i)
-    # end
-    # 
-    # def self.retrieve_by_tags_newer_than(event_id, tags, earliest_date, arguments={})
-    #   arguments["tags"] = tags.join(",")
-    #   arguments["min_upload_date"] = earliest_date.to_i + 1
-    #   arguments["sort"] = "date-posted-asc"
-    #   retrieve(event_id, arguments)
-    # end
-    # 
-    # private
-    # 
-    # ## search on flickr and add to db.
-    # def self.retrieve(event_id, arguments={})
-    #   puts "beginning retrieve"
-    #   response = request("flickr.photos.search",arguments)
-    #   # we need to grab the last looked up photo and get it's load date.
-    #   lastphoto = REXML::XPath.match(response, "//photo[last()]")
-    #   puts "about to find lastphotoid"
-    #   unless lastphoto[0]
-    #     puts "No images retrieved, stopping"
-    #     return
-    #   end
-    #   lastphotoid = lastphoto[0].attributes['id']
-    #   puts "should have lastphotoid."
-    #   # we will use this latest load date to set the loaded before in the db,
-    #   # to determine where to start the next load.
-    #   latest_load_date = posted_date_for_image(lastphotoid)
-    #   
-    #   # now stash them in DB.
-    #   photo_ids = REXML::XPath.match(response, "//photo").collect do |photoelem|
-    #     id = photoelem.attributes['id'].to_i
-    #     puts "Found image, id: #{id}"
-    #     fp = FlickrPhoto.new
-    #     fp.flickrid = id
-    #     fp.server = photoelem.attributes['server']
-    #     fp.farm = photoelem.attributes['farm']
-    #     fp.secret = photoelem.attributes['secret']
-    #     fp.title = photoelem.attributes['title']
-    #     fp.owner = photoelem.attributes['owner']
-    #     fp.posted_before = latest_load_date
-    #     fp.event_id = event_id
-    #     fp.save!
-    #   end # /attribute::id   a.value
-    # end
-    # 
-    # ## method for doing a request
-    # def self.request(method, arguments={})
-    #   api_key = APPLICATION_CONFIG.flickr_api_key
-    #   
-    #   endpoint = "http://api.flickr.com/services/rest/"
-    #   arg_map = arguments.collect {|arg,value| CGI.escape(arg)+"="+CGI.escape(value.to_s)}.join("&")
-    #   call_uri = "#{endpoint}?api_key=#{api_key}&method=#{method}&#{arg_map}"
-    #   response = REXML::Document.new(open(call_uri).read)
-    #   if response.root.attributes["stat"] == "ok" then
-    #     return response
-    #   elsif response.root.attributes["stat"] == "fail" then
-    #     raise "Flickr API error #{response.elements["rsp/err"].attributes["code"]}: #{response.elements["rsp/err"].attributes["msg"]}"
-    #   else
-    #     raise "Request status neither ok nor fail. Tried to fetch #{call_uri}."
-    #   end
-    # end
-    # 
-    # 
-
 end

config/environment.rb

@@ -25,6 +25,7 @@
   # config.gem "bj"
   # config.gem "hpricot", :version => '0.6', :source => "http://code.whytheluckystiff.net"
   # config.gem "aws-s3", :lib => "aws/s3"
+  config.gem "json"
 
   # Only load the plugins named here, in the order given. By default, all plugins 
   # in vendor/plugins are loaded in alphabetical order.

db/schema.rb

@@ -76,6 +76,20 @@
     t.datetime "updated_at"
   end
 
+  create_table "mugshots", :force => true do |t|
+    t.integer  "temp_user_id", :limit => 11
+    t.integer  "user_id",      :limit => 11
+    t.integer  "parent_id",    :limit => 11
+    t.string   "content_type"
+    t.string   "filename"
+    t.string   "thumbnail"
+    t.integer  "size",         :limit => 11
+    t.integer  "width",        :limit => 11
+    t.integer  "height",       :limit => 11
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
   create_table "open_id_authentication_associations", :force => true do |t|
     t.binary  "server_url"
     t.string  "handle"
@@ -86,9 +100,9 @@
   end
 
   create_table "open_id_authentication_nonces", :force => true do |t|
-    t.integer "timestamp",  :limit => 11, :null => false
+    t.integer "timestamp",  :limit => 11,                 :null => false
     t.string  "server_url"
-    t.string  "salt",                     :null => false
+    t.string  "salt",                     :default => "", :null => false
   end
 
   create_table "posts", :force => true do |t|

lib/tasks/fetcher.rake

@@ -4,6 +4,9 @@ require File.expand_path(File.dirname(__FILE__) + "/../../config/environment")
 
 namespace :fetcher do
 
+  desc 'Runs all fetch tasks'
+  task :all => [:flickr, :twitter]
+
   desc 'Fetches new images from flickr for all events'
   task :flickr do
     puts "Starting Flickr Fetch"
@@ -15,6 +18,11 @@ namespace :fetcher do
       end
       FlickrPhoto.retrieve_by_tags_newer_than(event.id, [event.tag], posted_before)
     end
-
+  end
+
+  desc 'Fetches new tweets from search.twitter for all events'
+  task :twitter do
+    puts "Starting tweet fetch"
+    Event.find(:all).each {|event| Tweet.retrieve(event)}
   end
 end