Merge pull request cloudfoundry#1397 from ljfranklin/PR-google-heavy-…

…stemcell-127591235

Include additional Infrastructures in heavy stemcell pipeline

bosh-dev/lib/bosh/dev/build.rb

@@ -41,6 +41,10 @@ def self.candidate(bucket_name='bosh-ci-pipeline')
       end
     end
 
+    def self.build_number
+      ENV.fetch('CANDIDATE_BUILD_NUMBER', '0000')
+    end
+
     def initialize(number, bucket_name, download_adapter, logger, skip_promote_artifacts, bearer_token)
       @number = number
       @logger = logger

bosh-dev/lib/bosh/dev/download_adapter.rb

@@ -1,4 +1,4 @@
-require 'net/http'
+require 'net/https'
 require 'uri'
 
 module Bosh::Dev
@@ -17,14 +17,8 @@ def download(uri, write_path)
     private
 
     def download_file(uri, write_path)
-      proxy = ENV['http_proxy'] ? URI.parse(ENV['http_proxy']) : NullUri.new
-      proxy = NullUri.new if bypass_proxy?(uri)
-
-      open_http_connection(proxy, uri) do |http|
-        http.read_timeout = 300
-        File.open(write_path, 'wb') do |file|
-          write_to_file(http, uri, file)
-        end
+      File.open(write_path, 'wb') do |file|
+        write_to_file(uri, file)
       end
     rescue Exception => e
       File.delete(write_path) if File.exist?(write_path)
@@ -33,9 +27,14 @@ def download_file(uri, write_path)
 
     def open_http_connection(proxy, uri, &block)
       tries = 0
+      http = nil
       begin
         http = Timeout.timeout(60) do
-          Net::HTTP.start(uri.host, uri.port, proxy.host, proxy.port, proxy.user, proxy.password)
+          http_opts = {read_timeout: 300}
+          if uri.scheme == 'https'
+            http_opts = http_opts.merge({use_ssl: true, verify_mode: OpenSSL::SSL::VERIFY_NONE})
+          end
+          Net::HTTP.start(uri.host, uri.port, proxy.host, proxy.port, proxy.user, proxy.password, http_opts)
         end
       rescue Timeout::Error => e
         @logger.info("Connecting to #{uri} timed out.")
@@ -46,39 +45,54 @@ def open_http_connection(proxy, uri, &block)
       end
       block.call(http)
     ensure
-      http.finish() if defined?(http)
+      http.finish() unless http.nil?
     end
 
-    def write_to_file(http, uri, file)
-      tries = 0
-      begin
-        headers = {}
-        headers['Range'] = "bytes=#{file.tell}-" if tries > 0
-        http.request_get(uri.request_uri, headers) do |response|
-          unless response.kind_of? Net::HTTPSuccess
-            raise "error #{response.code} while downloading '#{uri}'"
-          end
-
-          starting_byte = 0
-          starting_byte = response.content_range.first if response['Content-Range']
-          file.seek(starting_byte)
-          file.truncate(starting_byte)
-          if tries > 0
-            @logger.info("Resuming download of #{uri} from #{starting_byte} bytes")
-          end
+    def write_to_file(uri, file, redirects_remaining = 10)
+      proxy = ENV['http_proxy'] ? URI.parse(ENV['http_proxy']) : NullUri.new
+      proxy = NullUri.new if bypass_proxy?(uri)
 
-          response.read_body do |chunk|
-            file.write(chunk)
+      open_http_connection(proxy, uri) do |http|
+        tries = 0
+        begin
+          headers = {}
+          headers['Range'] = "bytes=#{file.tell}-" if tries > 0
+
+          http.request_get(uri.request_uri, headers) do |response|
+            if response.kind_of? Net::HTTPRedirection
+              if redirects_remaining > 0
+                return write_to_file(URI(response['location']), file, redirects_remaining - 1)
+              else
+                raise "infinite redirect loop while downloading '#{uri}'"
+              end
+            end
+            unless response.kind_of? Net::HTTPSuccess
+              err_msg = "error #{response.code} #{response.message} while downloading '#{uri}'"
+              err_msg += ": #{response.body}" if response.body
+              raise err_msg
+            end
+
+            starting_byte = 0
+            starting_byte = response.content_range.first if response['Content-Range']
+            file.seek(starting_byte)
+            file.truncate(starting_byte)
+            if tries > 0
+              @logger.info("Resuming download of #{uri} from #{starting_byte} bytes")
+            end
+
+            response.read_body do |chunk|
+              file.write(chunk)
+            end
           end
-        end
-      rescue Timeout::Error => e
-        @logger.info("Download of #{uri} timed out.")
+        rescue Timeout::Error => e
+          @logger.info("Download of #{uri} timed out.")
 
-        raise e unless tries < 3
-        tries += 1
+          raise e unless tries < 3
+          tries += 1
 
-        @logger.debug("Retrying ...")
-        retry
+          @logger.debug("Retrying ...")
+          retry
+        end
       end
     end
 

bosh-dev/lib/bosh/dev/stemcell_dependency_fetcher.rb

@@ -0,0 +1,39 @@
+require 'json'
+require 'uri'
+
+module Bosh::Dev
+  class StemcellDependencyFetcher
+    def initialize(downloader, logger)
+      @downloader = downloader
+      @logger = logger
+    end
+
+    def download_os_image(opts)
+      bucket_name = opts[:bucket_name]
+      key = opts[:key]
+      output_path = opts[:output_path]
+
+      os_image_versions_file = File.expand_path('../../../../../bosh-stemcell/os_image_versions.json', __FILE__)
+      os_image_versions = JSON.load(File.open(os_image_versions_file))
+      os_image_version = os_image_versions[key]
+      if os_image_version.nil?
+        raise "Unable to find OS image key '#{key}' in known versions: #{os_image_versions.to_json}"
+      end
+
+      os_image_uri = URI.join('https://s3.amazonaws.com/', "#{bucket_name}/", key)
+      os_image_uri.query = URI.encode_www_form([['versionId', os_image_version]])
+
+      @downloader.download(os_image_uri, output_path)
+    end
+
+    def download_bosh_micro_release(opts)
+      bosh_version = opts[:bosh_version]
+      output_path = opts[:output_path]
+
+      os_image_uri = URI.join('https://bosh.io/d/github.com/cloudfoundry/bosh')
+      os_image_uri.query = URI.encode_www_form([['v', bosh_version]])
+
+      @downloader.download(os_image_uri, output_path)
+    end
+  end
+end

bosh-dev/lib/bosh/dev/tasks/stemcell.rake

@@ -73,38 +73,99 @@ namespace :stemcell do
     puts "OS image #{args.os_image_path} version '#{file.version}' uploaded to S3 in bucket '#{args.s3_bucket_name}' with key '#{args.s3_bucket_key}'."
   end
 
-  desc 'Build a stemcell with a remote pre-built base OS image'
-  task :build, [:infrastructure_name, :hypervisor_name, :operating_system_name, :operating_system_version, :agent_name, :os_image_s3_bucket_name, :os_image_key] do |_, args|
+  desc 'Download a remote pre-built base OS image'
+  task :download_os_image, [:os_image_s3_bucket_name, :os_image_key] do |_, args|
     begin
-      require 'uri'
-      require 'tempfile'
       require 'bosh/dev/download_adapter'
+      require 'bosh/dev/stemcell_dependency_fetcher'
+
+      puts "Using OS image #{args.os_image_key} from #{args.os_image_s3_bucket_name}"
+
+      logger = Logging.logger($stdout)
+      downloader = Bosh::Dev::DownloadAdapter.new(logger)
+      fetcher = Bosh::Dev::StemcellDependencyFetcher.new(downloader, logger)
+
+      mkdir_p('tmp')
+      os_image_path = File.join(Dir.pwd, 'tmp', 'base_os_image.tgz')
+      fetcher.download_os_image(
+        bucket_name: args.os_image_s3_bucket_name,
+        key:         args.os_image_key,
+        output_path: os_image_path,
+      )
+
+      puts "Successfully downloaded OS image to #{os_image_path}"
+    rescue RuntimeError => e
+      print_help
+      raise e
+    end
+  end
+
+  PINNED_MICRO_VERSION = '257.3'
+
+  desc "Download a remote BOSH micro release, pinned to #{PINNED_MICRO_VERSION}"
+  task :download_bosh_micro_release do |_, args|
+    begin
+      require 'bosh/dev/download_adapter'
+      require 'bosh/dev/stemcell_dependency_fetcher'
+
+      puts "Downloading BOSH micro release version '#{PINNED_MICRO_VERSION}'"
+
+      logger = Logging.logger($stdout)
+      downloader = Bosh::Dev::DownloadAdapter.new(logger)
+      fetcher = Bosh::Dev::StemcellDependencyFetcher.new(downloader, logger)
+
+      mkdir_p('tmp')
+      release_path = File.join(Dir.pwd, 'tmp', "bosh-#{PINNED_MICRO_VERSION}.tgz")
+      fetcher.download_bosh_micro_release(
+        bosh_version: PINNED_MICRO_VERSION,
+        output_path: release_path,
+      )
 
-      os_image_versions_file = File.expand_path('../../../../../../bosh-stemcell/os_image_versions.json', __FILE__)
-      os_image_versions = JSON.load(File.open(os_image_versions_file))
-      os_image_version = os_image_versions[args.os_image_key]
-      puts "Using OS image #{args.os_image_key}, version #{os_image_version}"
+      puts "Successfully downloaded BOSH micro release to #{release_path}"
+    rescue RuntimeError => e
+      print_help
+      raise e
+    end
+  end
 
-      os_image_uri = URI.join('http://s3.amazonaws.com/', "#{args.os_image_s3_bucket_name}/", args.os_image_key)
-      os_image_uri.query = URI.encode_www_form([['versionId', os_image_version]])
+  desc 'Build a stemcell with a remote pre-built base OS image and bosh micro release'
+  task :build, [:infrastructure_name, :hypervisor_name, :operating_system_name, :operating_system_version, :agent_name, :os_image_s3_bucket_name, :os_image_key] do |_, args|
+    begin
+      require 'bosh/dev/download_adapter'
+      require 'bosh/dev/stemcell_dependency_fetcher'
 
-      Dir.mktmpdir('os-image') do |download_path|
-        os_image_path = File.join(download_path, 'base_os_image.tgz')
-        downloader = Bosh::Dev::DownloadAdapter.new(Logging.logger($stdout))
-        downloader.download(os_image_uri, os_image_path)
+      logger = Logging.logger($stdout)
+      downloader = Bosh::Dev::DownloadAdapter.new(logger)
+      fetcher = Bosh::Dev::StemcellDependencyFetcher.new(downloader, logger)
 
-        Rake::Task['stemcell:build_with_local_os_image'].invoke(args.infrastructure_name, args.hypervisor_name, args.operating_system_name, args.operating_system_version, args.agent_name, os_image_path)
+      mkdir_p('tmp')
+      if 'no' == ENV['BOSH_MICRO_ENABLED']
+        release_path = '/dev/null'
+      else
+        release_path = File.join(Dir.pwd, 'tmp', "bosh-#{PINNED_MICRO_VERSION}.tgz")
+        fetcher.download_bosh_micro_release(
+          bosh_version: PINNED_MICRO_VERSION,
+          output_path: release_path,
+        )
       end
+      os_image_path = File.join(Dir.pwd, 'tmp', 'base_os_image.tgz')
+      fetcher.download_os_image(
+        bucket_name: args.os_image_s3_bucket_name,
+        key:         args.os_image_key,
+        output_path: os_image_path,
+      )
 
+      Rake::Task['stemcell:build_with_local_os_image_with_bosh_release_tarball'].invoke(args.infrastructure_name, args.hypervisor_name, args.operating_system_name, args.operating_system_version, args.agent_name, os_image_path, release_path)
     rescue RuntimeError => e
       print_help
       raise e
     end
   end
 
-  desc 'Build a stemcell using a local OS image and release'
+  desc 'Build a stemcell using a local OS image and bosh micro release'
   task :build_with_local_os_image_with_bosh_release_tarball, [:infrastructure_name, :hypervisor_name, :operating_system_name, :operating_system_version, :agent_name, :os_image_path, :bosh_release_tarball_path, :build_number] do |_, args|
     begin
+      require 'bosh/dev/build'
       require 'bosh/dev/gem_components'
       require 'bosh/stemcell/build_environment'
       require 'bosh/stemcell/definition'
@@ -113,6 +174,8 @@ namespace :stemcell do
       require 'bosh/stemcell/stemcell_packager'
       require 'bosh/stemcell/stemcell_builder'
 
+      args.with_defaults(build_number: Bosh::Dev::Build.build_number)
+
       gem_components = Bosh::Dev::GemComponents.new(args.build_number)
       definition = Bosh::Stemcell::Definition.for(args.infrastructure_name, args.hypervisor_name, args.operating_system_name, args.operating_system_version, args.agent_name, false)
       environment = Bosh::Stemcell::BuildEnvironment.new(
@@ -172,16 +235,22 @@ namespace :stemcell do
   task :build_with_local_os_image, [:infrastructure_name, :hypervisor_name, :operating_system_name, :operating_system_version, :agent_name, :os_image_path] do |_, args|
 
     begin
-      require 'bosh/dev/build'
+      require 'bosh/dev/download_adapter'
+      require 'bosh/dev/stemcell_dependency_fetcher'
 
-      # download bosh release from bucket
-      build = Bosh::Dev::Build.candidate
-      build_number = build.number
+      logger = Logging.logger($stdout)
+      downloader = Bosh::Dev::DownloadAdapter.new(logger)
+      fetcher = Bosh::Dev::StemcellDependencyFetcher.new(downloader, logger)
 
+      mkdir_p('tmp')
       if 'no' == ENV['BOSH_MICRO_ENABLED']
-        release_tarball_path = '/dev/null'
+        release_path = '/dev/null'
       else
-        release_tarball_path = build.release_tarball_path
+        release_path = File.join(Dir.pwd, 'tmp', "bosh-#{PINNED_MICRO_VERSION}.tgz")
+        fetcher.download_bosh_micro_release(
+          bosh_version: PINNED_MICRO_VERSION,
+          output_path: release_path,
+        )
       end
     rescue RuntimeError => e
       print_help
@@ -195,8 +264,7 @@ namespace :stemcell do
       args.operating_system_version,
       args.agent_name,
       args.os_image_path,
-      release_tarball_path,
-      build_number
+      release_path,
     )
   end
 

bosh-dev/spec/unit/bosh/dev/build_spec.rb

@@ -9,6 +9,27 @@ module Bosh::Dev
 
     let(:bucket_name) { 'fake-bucket' }
 
+    describe '.build_number' do
+      subject { described_class.build_number }
+      before { stub_const('ENV', environment) }
+
+      context 'when CANDIDATE_BUILD_NUMBER is set' do
+        let(:environment) { { 'CANDIDATE_BUILD_NUMBER' => 'candidate'} }
+
+        it 'returns the specified build number' do
+          expect(subject).to eq('candidate')
+        end
+      end
+
+      context 'when CANDIDATE_BUILD_NUMBER is not set' do
+        let(:environment) { {} }
+
+        it 'returns the default build number' do
+          expect(subject).to eq('0000')
+        end
+      end
+    end
+
     describe '.candidate' do
       subject { described_class.candidate(bucket_name) }