Add wad support

.travis.yml

@@ -6,20 +6,34 @@ python:
   - "3.3"
   - "pypy"
 env:
-  - DJANGO=Django==1.2.7
-  - DJANGO=Django==1.3.7
-  - DJANGO=Django==1.4.10
-  - DJANGO=Django==1.5.5
-  - DJANGO=Django==1.6
-  - DJANGO="-e git+git://github.com/django/django.git#egg=Django"
+  matrix:
+    - DJANGO=Django==1.2.7
+    - DJANGO=Django==1.3.7
+    - DJANGO=Django==1.4.10
+    - DJANGO=Django==1.5.5
+    - DJANGO=Django==1.6
+    - DJANGO="-e git+git://github.com/django/django.git#egg=Django"
+  global:
+    - PIP_DOWNLOAD_CACHE=".pip_download_cache"
+    - WAD_FILES="package.json,Makefile,setup.py"
+    - WAD_INSTALL_COMMAND=ci/setup
 
+    # This should be specified with:
+    # travis encrypt S3_BUCKET_NAME=secretvalue S3_CREDENTIALS=accesskey:secretkey
+    # - S3_BUCKET_NAME=
+    # - S3_CREDENTIALS=
+    - secure: pZ7DtkEu2q/pGINfVT9S+iPRE5ck6mBQmuHTQz3PVXF/UJmpM1LCbC7aqrw4GXWuUhYc50QCnMZJL1yoBORVEfY4nfXHLRc3HMoWE6Srqf0IBDywi6T+UUdLeOYe13EDb4p8WIpnRGlV4WI1WaPXuFlMEdj5tAjlC9ZotLxyVaI=
+before_install:
+  # Use closer nameservers
+  - printf "nameserver 199.91.168.70\nnameserver 199.91.168.71\n" | sudo tee /etc/resolv.conf
+
+  # These need to be here and not in the env hash because they need to be
+  # evaluated after the virtualenv has been setup
+  - mkdir -p $PIP_DOWNLOAD_CACHE
+  - export WAD_ENVIRONMENT_VARIABLES="TRAVIS_PYTHON_VERSION,TRAVIS_NODE_VERSION,WAD_CACHE_PATH"
+  - export WAD_CACHE_PATH="node_modules,$PIP_DOWNLOAD_CACHE,$VIRTUAL_ENV"
 install:
-  - sudo apt-get install libevent-dev
-  - pip install $DJANGO --use-mirrors
-  - make bootstrap
-  - "if [[ ${TRAVIS_PYTHON_VERSION::1} == '2' ]]; then pip install gevent --use-mirrors; fi"
-  - "if [[ ${TRAVIS_PYTHON_VERSION} == '3.2' ]]; then pip install -I https://github.com/celery/celery/archive/3.0.zip; fi"
-  - "if [[ ${TRAVIS_PYTHON_VERSION::1} == '3' ]]; then pip uninstall django-celery -y; fi"
+  - time ci/wad
 script:
   - "if [[ ${TRAVIS_PYTHON_VERSION} != 'pypy' ]]; then make lint; fi"
   - py.test tests/ --cov raven --cov-report term-missing

ci/setup

@@ -0,0 +1,14 @@
+#!/bin/bash -eu
+
+sudo apt-get install libevent-dev
+pip install $DJANGO --use-mirrors
+make bootstrap
+if [[ ${TRAVIS_PYTHON_VERSION::1} == '2' ]]; then
+	pip install gevent --use-mirrors
+fi
+if [[ ${TRAVIS_PYTHON_VERSION} == '3.2' ]]; then
+	pip install -I https://github.com/celery/celery/archive/3.0.zip
+fi
+if [[ ${TRAVIS_PYTHON_VERSION::1} == '3' ]]; then
+	pip uninstall django-celery -y
+fi

ci/wad

@@ -0,0 +1,334 @@
+#!/usr/bin/env ruby
+
+# Generated on: 20-09-2013 at 12:38
+
+require 'time'
+require 'net/http'
+require 'net/https'
+require 'digest/md5'
+require 'digest/sha1'
+require 'fileutils'
+require 'openssl'
+require 'base64'
+require 'cgi'
+class Presss
+  # Computes the Authorization header for a AWS request based on a message,
+  # the access key ID and secret access key.
+  class Authorization
+    attr_accessor :access_key_id, :secret_access_key
+
+    def initialize(access_key_id, secret_access_key)
+      @access_key_id, @secret_access_key = access_key_id, secret_access_key
+    end
+
+    # Returns the value for the Authorization header for a message contents.
+    def header(string)
+      'AWS ' + access_key_id + ':' + sign(string)
+    end
+
+    # Returns a signature for a AWS request message.
+    def sign(string)
+      Base64.encode64(hmac_sha1(string)).strip
+    end
+
+    def hmac_sha1(string)
+      OpenSSL::HMAC.digest('sha1', secret_access_key, string)
+    end
+  end
+
+  class HTTP
+    attr_accessor :config
+
+    def initialize(config)
+      @config = config
+    end
+
+    # Returns the configured bucket name.
+    def bucket_name
+      config[:bucket_name]
+    end
+
+    def region
+      config[:region] || 'us-east-1'
+    end
+
+    def domain
+      case region
+      when 'us-east-1'
+        's3.amazonaws.com'
+      else
+        's3-%s.amazonaws.com' % region
+      end
+    end
+
+    def bucket_in_hostname?
+      config[:bucket_in_hostname]
+    end
+
+    def url_prefix
+      if bucket_in_hostname?
+        "https://#{bucket_name}.#{domain}"
+      else
+        "https://#{domain}/#{bucket_name}"
+      end
+    end
+
+    # Returns the absolute path based on the key for the object.
+    def absolute_path(path)
+      path.start_with?('/') ? path : '/' + path
+    end
+
+    # Returns the canonicalized resource used in the authorization
+    # signature for an absolute path to an object.
+    def canonicalized_resource(path)
+      if bucket_name.nil?
+        raise ArgumentError, "Please configure a bucket_name: Presss.config = { bucket_name: 'my-bucket-name }"
+      else
+        '/' + bucket_name + absolute_path(path)
+      end
+    end
+
+    # Returns a Presss::Authorization instance for the configured
+    # AWS credentials.
+    def authorization
+      @authorization ||= Presss::Authorization.new(
+        config[:access_key_id],
+        config[:secret_access_key]
+      )
+    end
+
+    def signed_url(verb, expires, headers, path)
+      path           = absolute_path(path)
+      canonical_path = canonicalized_resource(path)
+      signature      = [ verb.to_s.upcase, nil, nil, expires, [ headers, canonical_path ].flatten.compact ].flatten.join("\n")
+      signed         = authorization.sign(signature)
+      "#{url_prefix}#{path}?Signature=#{CGI.escape(signed)}&Expires=#{expires}&AWSAccessKeyId=#{CGI.escape(authorization.access_key_id)}"
+    end
+
+    def download(path, destination)
+      url = signed_url(:get, Time.now.to_i + 600, nil, path)
+      Presss.log "signed_url=#{url}"
+      system 'curl', '-f', '-S', '-o', destination, url
+      $?.success?
+    end
+
+    # Puts an object with a key using a file or string. Optionally pass in
+    # the content-type if you want to set a specific one.
+    def put(path, file)
+      header = 'x-amz-storage-class:REDUCED_REDUNDANCY'
+      url = signed_url(:put, Time.now.to_i + 600, header, path)
+      Presss.log "signed_url=#{url}"
+      system 'curl', '-f', '-S', '-H', header, '-T', file, url
+      $?.success?
+    end
+  end
+
+  class << self
+    attr_accessor :config
+    attr_accessor :logger
+  end
+  self.config = {}
+
+  # Get a object with a certain key.
+  def self.download(path, destination)
+    t0 = Time.now
+    request = Presss::HTTP.new(config)
+    log("Trying to GET #{path}")
+    if request.download(path, destination)
+      puts("[wad] Downloaded in #{(Time.now - t0).to_i} seconds")
+      true
+    else
+      nil
+    end
+  end
+
+  # Puts an object with a key using a file or string. Optionally pass in
+  # the content-type if you want to set a specific one.
+  def self.put(path, filename, content_type='application/x-download')
+    request = Presss::HTTP.new(config)
+    log("Trying to PUT #{path}")
+    request.put(path, filename)
+  end
+
+  # Logs to the configured logger if a logger was configured.
+  def self.log(message)
+    if logger
+      logger.info('[Presss] ' + message)
+    end
+  end
+end
+
+# Utility class to push and fetch Bundler directories to speed up
+# test runs on Travis-CI
+class Wad
+  class Key
+    def default_environment_variables
+      []
+    end
+
+    def default_files
+      [ "#{ENV['BUNDLE_GEMFILE']}.lock" ]
+    end
+
+    def environment_variables
+      if ENV['WAD_ENVIRONMENT_VARIABLES']
+        ENV['WAD_ENVIRONMENT_VARIABLES'].split(',')
+      else
+        default_environment_variables
+      end
+    end
+
+    def files
+      ENV['WAD_FILES'] ? ENV['WAD_FILES'].split(',') : default_files
+    end
+
+    def environment_variable_contents
+      environment_variables.map { |v| ENV[v] }
+    end
+
+    def file_contents
+      files.map { |f| File.read(f) rescue nil }
+    end
+
+    def contents
+      segments = [ RUBY_VERSION, RUBY_PLATFORM ] + environment_variable_contents + file_contents
+      Digest::SHA1.hexdigest(segments.join("\n"))
+    end
+  end
+
+  def initialize
+    s3_configure
+  end
+
+  def project_root
+    Dir.pwd
+  end
+
+  def artifact_name
+    @artifact_name ||= Key.new.contents
+  end
+
+  def bzip_filename
+    File.join(project_root, "tmp/#{artifact_name}.tar.bz2")
+  end
+
+  def cache_path
+    ENV['WAD_CACHE_PATH'] ? ENV['WAD_CACHE_PATH'].split(",") : [ '.bundle' ]
+  end
+
+  def s3_bucket_name
+    if bucket = ENV['WAD_S3_BUCKET_NAME'] || ENV['S3_BUCKET_NAME']
+      bucket
+    end
+  end
+
+  def s3_credentials
+    if creds = ENV['WAD_S3_CREDENTIALS'] || ENV['S3_CREDENTIALS']
+      creds.split(':')
+    end
+  end
+
+  def s3_access_key_id
+    s3_credentials && s3_credentials[0]
+  end
+
+  def s3_secret_access_key
+    s3_credentials && s3_credentials[1]
+  end
+
+  def s3_path
+    "#{artifact_name}.tar.bz2"
+  end
+
+  def s3_configure
+    Presss.config = {
+      :bucket_name => s3_bucket_name,
+      :access_key_id => s3_access_key_id,
+      :secret_access_key => s3_secret_access_key,
+      :region => ENV['WAD_AWS_REGION'],
+      :bucket_in_hostname => (ENV['WAD_BUCKET_IN_HOSTNAME'] == 'true')
+    }
+  end
+
+  def s3_write
+    log "Trying to write Wad to S3"
+    if Presss.put(s3_path, bzip_filename)
+      log "Wrote Wad to S3"
+    else
+      log "Failed to write to S3, debug with `wad -v'"
+    end
+  end
+
+  def s3_read
+    if File.exist?(bzip_filename)
+      log "Removing bundle from filesystem"
+      FileUtils.rm_f(bzip_filename)
+    end
+
+    log "Trying to fetch Wad from S3"
+    FileUtils.mkdir_p(File.dirname(bzip_filename))
+    Presss.download(s3_path, bzip_filename)
+  end
+
+  def zip
+    log "Creating artifact with tar (#{File.basename(bzip_filename)})"
+    system("cd #{project_root} && tar -cPjf #{bzip_filename} #{cache_path.join(' ')}")
+    $?.success?
+  end
+
+
+  def unzip
+    log "Unpacking artifact with tar (#{File.basename(bzip_filename)})"
+    system("cd #{project_root} && tar -xPjf #{bzip_filename}")
+    $?.success?
+  end
+
+  def put
+    zip
+    s3_write
+  end
+
+  def get
+    if s3_read
+      unzip
+    end
+  end
+
+  def default_command
+    bundle_without = ENV['WAD_BUNDLE_WITHOUT'] || "development production"
+    "bundle install --path .bundle --without='#{bundle_without}'"
+  end
+
+  def install
+    log "Installing..."
+    command = ENV['WAD_INSTALL_COMMAND'] || default_command
+    puts command
+    system(command)
+    $?.success?
+  end
+
+  def setup
+    if !s3_credentials || !s3_bucket_name
+      log "No S3 credentials defined. Set WAD_S3_CREDENTIALS= and WAD_S3_BUCKET_NAME= for caching."
+      install
+    elsif get
+      install
+    elsif install
+      put
+    else
+      abort "Failed properly fetch or install. Please review the logs."
+    end
+  end
+
+  def log(message)
+    puts "[wad] #{message}"
+  end
+end
+
+if ARGV.index('-v')
+  require 'logger'
+  Presss.logger = Logger.new($stdout)
+end
+
+Wad.new.setup
+__END__