extracted job template rendering into bosh-director-core gem

Signed-off-by: Mike Stallard <[email protected]>

Gemfile

@@ -16,6 +16,7 @@ gemspec path: 'bosh_openstack_cpi'
 gemspec path: 'bosh-registry'
 gemspec path: 'bosh_vsphere_cpi'
 gemspec path: 'bosh-director'
+gemspec path: 'bosh-director-core'
 gemspec path: 'bosh-monitor'
 gemspec path: 'bosh-release'
 gemspec path: 'simple_blobstore_server'

Gemfile.lock

@@ -42,6 +42,7 @@ PATH
       bcrypt-ruby (~> 3.0.1)
       blobstore_client (~> 1.1840.0)
       bosh-core (~> 1.1840.0)
+      bosh-director-core (~> 1.1840.0)
       bosh_aws_cpi (~> 1.1840.0)
       bosh_common (~> 1.1840.0)
       bosh_cpi (~> 1.1840.0)
@@ -64,6 +65,12 @@ PATH
       thin (~> 1.5.0)
       yajl-ruby (~> 1.1.0)
 
+PATH
+  remote: bosh-director-core
+  specs:
+    bosh-director-core (1.1836.0)
+      bosh_common (~> 1.1836.0)
+
 PATH
   remote: bosh-monitor
   specs:
@@ -416,6 +423,7 @@ DEPENDENCIES
   bosh-core!
   bosh-dev!
   bosh-director!
+  bosh-director-core!
   bosh-monitor!
   bosh-registry!
   bosh-release!

bosh-dev/lib/bosh/dev/gem_components.rb

@@ -37,6 +37,7 @@ def each(&block)
        bosh-registry
        bosh_vsphere_cpi
        bosh-director
+       bosh-director-core
        bosh-monitor
        bosh-release
        simple_blobstore_server

bosh-dev/spec/bosh/dev/gem_components_spec.rb

@@ -41,6 +41,7 @@ module Bosh::Dev
         GemComponent.should_receive(:new).with('bosh-registry', gem_version.version)
         GemComponent.should_receive(:new).with('bosh_vsphere_cpi', gem_version.version)
         GemComponent.should_receive(:new).with('bosh-director', gem_version.version)
+        GemComponent.should_receive(:new).with('bosh-director-core', gem_version.version)
         GemComponent.should_receive(:new).with('bosh-monitor', gem_version.version)
         GemComponent.should_receive(:new).with('bosh-release', gem_version.version)
         GemComponent.should_receive(:new).with('simple_blobstore_server', gem_version.version)
@@ -67,6 +68,7 @@ module Bosh::Dev
           bosh-registry
           bosh_vsphere_cpi
           bosh-director
+          bosh-director-core
           bosh-monitor
           bosh-release
           simple_blobstore_server

bosh-director-core/.rspec

@@ -0,0 +1,4 @@
+--color
+--profile
+--require rspec/instafail
+--format RSpec::Instafail

bosh-director-core/bosh-director-core.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+require File.expand_path('../lib/bosh/director/core/version', __FILE__)
+
+version = Bosh::Director::Core::VERSION
+
+Gem::Specification.new do |spec|
+  spec.name        = 'bosh-director-core'
+  spec.version     = version
+  spec.authors     = 'Pivotal'
+  spec.email       = '[email protected]'
+  spec.description = 'Bosh::Director::Core provides common Director code for Director and Microbosh Deployer'
+  spec.summary     = 'Bosh::Director::Core provides common Director code for Director and Microbosh Deployer'
+  spec.homepage    = 'https://github.com/cloudfoundry/bosh'
+  spec.license     = 'Apache 2.0'
+
+  spec.required_ruby_version = Gem::Requirement.new('>= 1.9.3')
+
+  spec.files         = `git ls-files -- lib/*`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = %w[lib]
+
+  spec.add_dependency 'bosh_common', "~>#{version}"
+
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'fakefs'
+end

bosh-director-core/lib/bosh/director/core.rb

@@ -0,0 +1,6 @@
+module Bosh
+  module Director
+    module Core
+    end
+  end
+end

bosh-director/lib/bosh/director/tar_gzipper.rb → bosh-director-core/lib/bosh/director/core/tar_gzipper.rb

@@ -1,29 +1,24 @@
 require 'tmpdir'
 require 'open3'
 
-module Bosh::Director
+module Bosh::Director::Core
   class TarGzipper
     # @param [String] base_dir the directory from which the tar command is run
     # @param [String, Array] sources the relative paths to include
     # @param [String] dest the destination filename for the tgz output
     # @param [Hash] options the options for compress
     # @option options [Boolean] :copy_first copy the source to a temp dir before archiving
-    def compress(base_dir, sources, dest, options={})
+    def compress(base_dir, sources, dest, options = {})
       sources = [*sources]
-
       sources.each do |source|
-        raise "Sources must have a path depth of 1 and contain no '#{File::SEPARATOR}'" if source.include?(File::SEPARATOR)
+        if source.include?(File::SEPARATOR)
+          raise "Sources must have a path depth of 1 and contain no '#{File::SEPARATOR}'"
+        end
       end
 
       base_dir_path = Pathname.new(base_dir)
-
-      unless base_dir_path.exist?
-        raise "The base directory #{base_dir} could not be found."
-      end
-
-      unless base_dir_path.absolute?
-        raise "The base directory #{base_dir} is not an absolute path."
-      end
+      raise "The base directory #{base_dir} could not be found." unless base_dir_path.exist?
+      raise "The base directory #{base_dir} is not an absolute path." unless base_dir_path.absolute?
 
       if options[:copy_first]
         Dir.mktmpdir do |tmpdir|

bosh-director-core/lib/bosh/director/core/templates.rb

@@ -0,0 +1,6 @@
+require 'bosh/director/core'
+
+module Bosh::Director::Core
+  module Templates
+  end
+end

bosh-director/lib/bosh/director/compressed_rendered_job_templates.rb → bosh-director-core/lib/bosh/director/core/templates/compressed_rendered_job_templates.rb

@@ -1,9 +1,10 @@
 require 'tmpdir'
 require 'digest/sha1'
-require 'bosh/director/tar_gzipper'
-require 'bosh/director/rendered_templates_writer'
+require 'bosh/director/core/templates'
+require 'bosh/director/core/tar_gzipper'
+require 'bosh/director/core/templates/rendered_templates_writer'
 
-module Bosh::Director
+module Bosh::Director::Core::Templates
   class CompressedRenderedJobTemplates
     def initialize(path)
       @path = path
@@ -14,7 +15,7 @@ def write(rendered_templates)
         writer = RenderedTemplatesWriter.new
         writer.write(rendered_templates, dir)
 
-        tar_gzipper = TarGzipper.new
+        tar_gzipper = Bosh::Director::Core::TarGzipper.new
         tar_gzipper.compress(dir, %w(.), @path)
       end
     end

bosh-director/lib/bosh/director/job_instance_renderer.rb → bosh-director-core/lib/bosh/director/core/templates/job_instance_renderer.rb

@@ -1,6 +1,7 @@
-require 'bosh/director/rendered_job_instance'
+require 'bosh/director/core/templates'
+require 'bosh/director/core/templates/rendered_job_instance'
 
-module Bosh::Director
+module Bosh::Director::Core::Templates
   class JobInstanceRenderer
     def initialize(job, job_template_loader)
       @job = job

bosh-director/lib/bosh/director/job_template_loader.rb → bosh-director-core/lib/bosh/director/core/templates/job_template_loader.rb

@@ -1,8 +1,8 @@
-require 'bosh/director/job_template_renderer'
-
-module Bosh::Director
-  SrcFileTemplate = Struct.new(:src_name, :dest_name, :erb_file)
+require 'bosh/director/core/templates'
+require 'bosh/director/core/templates/job_template_renderer'
+require 'bosh/director/core/templates/src_file_template'
 
+module Bosh::Director::Core::Templates
   class JobTemplateLoader
     def initialize(logger)
       @logger = logger

bosh-director/lib/bosh/director/job_template_renderer.rb → bosh-director-core/lib/bosh/director/core/templates/job_template_renderer.rb

@@ -1,7 +1,9 @@
-require 'bosh/director/rendered_job_template'
+require 'bosh/director/core/templates'
+require 'bosh/director/core/templates/rendered_job_template'
+require 'bosh/director/core/templates/rendered_file_template'
+require 'common/properties'
 
-module Bosh::Director
-  RenderedFileTemplate = Struct.new(:src_name, :dest_name, :contents)
+module Bosh::Director::Core::Templates
   class JobTemplateRenderer
 
     attr_reader :monit_template, :templates
@@ -32,6 +34,7 @@ def render(job_name, instance)
 
     def render_erb(job_name, template, template_context, index)
       template.result(template_context.get_binding)
+    # rubocop:disable RescueException
     rescue Exception => e
       logger.debug(e.inspect)
       job_desc = "#{job_name}/#{index}"
@@ -43,8 +46,8 @@ def render_erb(job_name, template, template_context, index)
         "for `#{job_desc}' (line #{line}: #{e})"
 
       logger.debug("#{message}\n#{e.backtrace.join("\n")}")
-
-      raise JobTemplateBindingFailed, "#{message}"
+      raise message
     end
+    # rubocop:enable RescueException
   end
 end

bosh-director-core/lib/bosh/director/core/templates/rendered_file_template.rb

@@ -0,0 +1,5 @@
+require 'bosh/director/core/templates'
+
+module Bosh::Director::Core::Templates
+  RenderedFileTemplate = Struct.new(:src_name, :dest_name, :contents)
+end

bosh-director/lib/bosh/director/rendered_job_instance.rb → bosh-director-core/lib/bosh/director/core/templates/rendered_job_instance.rb

@@ -1,9 +1,10 @@
-require 'bosh/director/deployment_plan/rendered_templates_archive'
-require 'bosh/director/compressed_rendered_job_templates'
+require 'bosh/director/core/templates'
+require 'bosh/director/core/templates/rendered_templates_archive'
+require 'bosh/director/core/templates/compressed_rendered_job_templates'
 require 'digest/sha1'
 require 'tempfile'
 
-module Bosh::Director
+module Bosh::Director::Core::Templates
   class RenderedJobInstance
     def initialize(job_templates)
       @job_templates = job_templates
@@ -37,12 +38,13 @@ def persist(blobstore)
       compressed_archive.write(job_templates)
 
       blobstore_id = blobstore.create(compressed_archive.contents)
-      DeploymentPlan::RenderedTemplatesArchive.new(blobstore_id, compressed_archive.sha1)
+      RenderedTemplatesArchive.new(blobstore_id, compressed_archive.sha1)
     ensure
       file.close!
     end
 
     private
+
     attr_reader :job_templates
   end
 end

bosh-director/lib/bosh/director/rendered_job_template.rb → bosh-director-core/lib/bosh/director/core/templates/rendered_job_template.rb

@@ -1,4 +1,6 @@
-module Bosh::Director
+require 'bosh/director/core/templates'
+
+module Bosh::Director::Core::Templates
   class RenderedJobTemplate
     attr_reader :name, :monit, :templates
 

bosh-director/lib/bosh/director/deployment_plan/rendered_templates_archive.rb → bosh-director-core/lib/bosh/director/core/templates/rendered_templates_archive.rb

@@ -1,4 +1,6 @@
-module Bosh::Director::DeploymentPlan
+require 'bosh/director/core/templates'
+
+module Bosh::Director::Core::Templates
   class RenderedTemplatesArchive
     attr_reader :blobstore_id, :sha1
 

bosh-director/lib/bosh/director/rendered_templates_writer.rb → bosh-director-core/lib/bosh/director/core/templates/rendered_templates_writer.rb

@@ -1,6 +1,7 @@
+require 'bosh/director/core/templates'
 require 'fileutils'
 
-module Bosh::Director
+module Bosh::Director::Core::Templates
   class RenderedTemplatesWriter
     def write(rendered_templates, output_dir)
       rendered_templates.each do |job_template|

bosh-director-core/lib/bosh/director/core/templates/src_file_template.rb

@@ -0,0 +1,5 @@
+require 'bosh/director/core/templates'
+
+module Bosh::Director::Core::Templates
+  SrcFileTemplate = Struct.new(:src_name, :dest_name, :erb_file)
+end

bosh-director-core/lib/bosh/director/core/version.rb

@@ -0,0 +1,7 @@
+module Bosh
+  module Director
+    module Core
+      VERSION = '1.1840.0'
+    end
+  end
+end

bosh-director-core/spec/bosh/director/core/tar_gzipper_spec.rb

@@ -0,0 +1,88 @@
+require 'spec_helper'
+
+require 'tmpdir'
+require 'fileutils'
+require 'bosh/director/core/tar_gzipper'
+
+module Bosh::Director::Core
+  describe TarGzipper do
+    let(:base_dir) { Dir.mktmpdir }
+    let(:sources) { %w(1 one) }
+    let(:dest) { Tempfile.new('logs') } # must keep tempfile reference lest it rm
+    let(:errored_retval) { ['stdout string', 'a stderr message', double('Status', success?: false, exitstatus: 5)] }
+    let(:success_retval) { ['', '', double('Status', success?: true)] }
+
+    before do
+      path = File.join(base_dir, '1', '2')
+      FileUtils.mkdir_p(path)
+      File.write(File.join(path, 'hello.log'), 'hello')
+
+      path = File.join(base_dir, 'one', 'two')
+      FileUtils.mkdir_p(path)
+      File.write(File.join(path, 'goodbye.log'), 'goodbye')
+    end
+
+    after do
+      FileUtils.rm_rf(base_dir)
+      FileUtils.rm_rf(dest.path)
+    end
+
+    context 'when copy first feature is enabled' do
+      it 'copies the files to a temp directory before archiving' do
+        Dir.stub(:mktmpdir).and_yield('/tempthing')
+        FileUtils.should_receive(:cp_r).with(%w(/foo/baz /foo/bar), '/tempthing/')
+        Pathname.stub(:new).and_return(instance_double('Pathname', exist?: true, absolute?: true))
+
+        Open3.should_receive(:capture3)
+        .with(*%w(tar -C /tempthing -czf /tmp/backup.tgz baz bar))
+        .and_return(success_retval)
+
+        subject.compress('/foo', %w(baz bar), '/tmp/backup.tgz', copy_first: true)
+      end
+    end
+
+    it 'raises when a source has a path depth greater than 1' do # sources that contain a '/'
+      expect {
+        subject.compress('/var/vcap/foo', %w(foo/bar), '/tmp/backup.tgz')
+      }.to raise_error("Sources must have a path depth of 1 and contain no '/'")
+    end
+
+    context 'if the source directory does not exist' do
+      let(:base_dir) { '/tmp/this/is/not/here' }
+
+      it 'raises an error' do
+        FileUtils.rm_rf(base_dir)
+
+        expect {
+          subject.compress(base_dir, sources, dest.path)
+        }.to raise_error("The base directory #{base_dir} could not be found.")
+      end
+    end
+
+    context 'if the base_dir is not absolute' do
+      let(:base_dir) { 'tmp' }
+
+      it 'raises an error' do
+        expect {
+          subject.compress(base_dir, sources, dest.path)
+        }.to raise_error("The base directory #{base_dir} is not an absolute path.")
+      end
+    end
+
+    it 'packages the sources into the destination tarball' do
+      subject.compress(base_dir, sources, dest.path)
+
+      output = `tar tzvf #{dest.path}`
+      expect(output).to include(' 1/2/hello.log')
+      expect(output).to include(' one/two/goodbye.log')
+    end
+
+    it 'raises if it fails to tar' do
+      Open3.stub(:capture3).and_return(errored_retval)
+      expect {
+        subject.compress(base_dir, sources, dest)
+      }.to raise_error(RuntimeError,
+                       "tar exited 5, output: 'stdout string', error: 'a stderr message'")
+    end
+  end
+end