Merge pull request waynehoover#23 from ericboehs/rake_cleaner

Rake task to clean old uploads

README.md

@@ -180,6 +180,33 @@ $(document).bind 's3_uploads_complete', ->
     alert("All Uploads completed")
 ```
 
+## Cleaning old uploads on S3
+You may be processing the files upon upload and reuploading them to another
+bucket or directory. If so you can remove the originali files by running a
+rake task.
+
+First, add the fog gem to your `Gemfile` and run `bundle`:
+```ruby
+  require 'fog'
+```
+
+Then, run the rake task to delete uploads older than 2 days:
+```
+  $ rake s3_direct_upload:clean_remote_uploads
+  Deleted file with key: "uploads/20121210T2139Z_03846cb0329b6a8eba481ec689135701/06 - PCR_RYA014-25.jpg"
+  Deleted file with key: "uploads/20121210T2139Z_03846cb0329b6a8eba481ec689135701/05 - PCR_RYA014-24.jpg"
+  $
+```
+
+Optionally customize the prefix used for cleaning (default is `uploads/#{2.days.ago.strftime('%Y%m%d')}`):
+**config/initalizers/s3_direct_upload.rb**
+```ruby
+S3DirectUpload.config do |c|
+  # ...
+  c.prefix_to_clean = "my_path/#{1.week.ago.strftime('%y%m%d')}"
+end
+```
+
 ## Contributing / TODO
 This is just a simple gem that only really provides some javascript and a form helper. 
 This gem could go all sorts of ways based on what people want and how people contribute. 

lib/s3_direct_upload/config_aws.rb

@@ -4,7 +4,7 @@ module S3DirectUpload
   class Config
     include Singleton
 
-    ATTRIBUTES = [:access_key_id, :secret_access_key, :bucket]
+    ATTRIBUTES = [:access_key_id, :secret_access_key, :bucket, :prefix_to_clean]
 
     attr_accessor *ATTRIBUTES
   end
@@ -15,4 +15,4 @@ def self.config
     end
     Config.instance
   end
-end
+end

lib/s3_direct_upload/form_helper.rb

@@ -49,7 +49,7 @@ def fields
       end
 
       def key
-        @key ||= "uploads/#{SecureRandom.hex}/${filename}"
+        @key ||= "uploads/#{DateTime.now.utc.strftime("%Y%m%dT%H%MZ")}_#{SecureRandom.hex}/${filename}"
       end
 
       def url
@@ -85,4 +85,4 @@ def signature
       end
     end
   end
-end
+end

lib/tasks/s3_direct_upload.rake

@@ -0,0 +1,57 @@
+namespace :s3_direct_upload do
+  desc "Removes old uploads from specified s3 bucket/directory -- Useful when uploads are processed into another directory"
+  task :clean_remote_uploads do
+    require 'thread'
+    require 'fog'
+
+    s3     = Fog::Storage::AWS.new(aws_access_key_id: S3DirectUpload.config.access_key_id, aws_secret_access_key: S3DirectUpload.config.secret_access_key)
+    bucket = S3DirectUpload.config.bucket
+    prefix = S3DirectUpload.config.prefix_to_clean || "uploads/#{2.days.ago.strftime('%Y%m%d')}"
+
+    queue         = Queue.new
+    semaphore     = Mutex.new
+    threads       = []
+    thread_count  = 20
+    total_listed  = 0
+    total_deleted = 0
+
+    threads << Thread.new do
+      Thread.current[:name] = "get files"
+      # Get all the files from this bucket. Fog handles pagination internally.
+      s3.directories.get("#{bucket}").files.all({prefix: prefix}).each do |file|
+        queue.enq(file)
+        total_listed += 1
+      end
+      # Add a final EOF message to signal the deletion threads to stop.
+      thread_count.times { queue.enq(:EOF) }
+    end
+
+    # Delete all the files in the queue until EOF with N threads.
+    thread_count.times do |count|
+      threads << Thread.new(count) do |number|
+        Thread.current[:name] = "delete files(#{number})"
+        # Dequeue until EOF.
+        file = nil
+        while file != :EOF
+          # Dequeue the latest file and delete it. (Will block until it gets a new file.)
+          file = queue.deq
+          unless file == :EOF
+            file.destroy
+            puts %Q{Deleted file with key: "#{file.key}"}
+          end
+          # Increment the global synchronized counter.
+          semaphore.synchronize {total_deleted += 1}
+        end
+      end
+    end
+
+    # Wait for the threads to finish.
+    threads.each do |t|
+      begin
+        t.join
+      rescue RuntimeError => e
+        puts "Failure on thread #{t[:name]}: #{e.message}"
+      end
+    end
+  end
+end