(PUP-2758) Add benchmarks empty catalog, and defined_types for future

This adds benchmark that runs the defined_types benchmark for the future
parser (everything else is the same).
A benchmark is also added to measure the overhead of compiling a
catalog with minimal (almost nothing) content (i.e. a command line
puppet apply scenario).

benchmarks/defined_types4/benchmarker.rb

@@ -0,0 +1,73 @@
+require 'erb'
+require 'ostruct'
+require 'fileutils'
+require 'json'
+
+class Benchmarker
+  include FileUtils
+
+  def initialize(target, size)
+    @target = target
+    @size = size
+  end
+
+  def setup
+    require 'puppet'
+    config = File.join(@target, 'puppet.conf')
+    Puppet.initialize_settings(['--config', config])
+  end
+
+  def run(args=nil)
+    env = Puppet.lookup(:environments).get('benchmarking')
+    node = Puppet::Node.new("testing", :environment => env)
+    Puppet::Resource::Catalog.indirection.find("testing", :use_node => node)
+  end
+
+  def generate
+    environment = File.join(@target, 'environments', 'benchmarking')
+    templates = File.join('benchmarks', 'defined_types4')
+
+    mkdir_p(File.join(environment, 'modules'))
+    mkdir_p(File.join(environment, 'manifests'))
+
+    render(File.join(templates, 'site.pp.erb'),
+           File.join(environment, 'manifests', 'site.pp'),
+           :size => @size)
+
+    @size.times do |i|
+      module_name = "module#{i}"
+      module_base = File.join(environment, 'modules', module_name)
+      manifests = File.join(module_base, 'manifests')
+
+      mkdir_p(manifests)
+
+      File.open(File.join(module_base, 'metadata.json'), 'w') do |f|
+        JSON.dump({
+          "types" => [],
+          "source" => "",
+          "author" => "Defined Types Benchmark Future Parser",
+          "license" => "Apache 2.0",
+          "version" => "1.0.0",
+          "description" => "Defined Types benchmark module #{i}",
+          "summary" => "Just this benchmark module, you know?",
+          "dependencies" => [],
+        }, f)
+      end
+
+      render(File.join(templates, 'module', 'testing.pp.erb'),
+             File.join(manifests, 'testing.pp'),
+             :name => module_name)
+    end
+
+    render(File.join(templates, 'puppet.conf.erb'),
+           File.join(@target, 'puppet.conf'),
+           :location => @target)
+  end
+
+  def render(erb_file, output_file, bindings)
+    site = ERB.new(File.read(erb_file))
+    File.open(output_file, 'w') do |fh|
+      fh.write(site.result(OpenStruct.new(bindings).instance_eval { binding }))
+    end
+  end
+end

benchmarks/defined_types4/description

@@ -0,0 +1,4 @@
+Benchmark scenario: heavy use of defined types future parser
+Benchmark target: catalog compilation
+Parser: Future
+

benchmarks/defined_types4/module/testing.pp.erb

@@ -0,0 +1,3 @@
+define <%= name %>::testing {
+  notify { "in <%= name %>: $title": }
+}

benchmarks/defined_types4/puppet.conf.erb

@@ -0,0 +1,4 @@
+confdir = <%= location %>
+vardir = <%= location %>
+environmentpath = <%= File.join(location, 'environments') %>
+parser = future

benchmarks/defined_types4/site.pp.erb

@@ -0,0 +1,4 @@
+<% size.times do |i| %>
+  module<%= i %>::testing { "first": }
+  module<%= i %>::testing{ "second": }
+<% end %>

benchmarks/empty_catalog/benchmarker.rb

@@ -0,0 +1,47 @@
+require 'erb'
+require 'ostruct'
+require 'fileutils'
+require 'json'
+
+class Benchmarker
+  include FileUtils
+
+  def initialize(target, size)
+    @target = target
+    @size = size
+  end
+
+  def setup
+    require 'puppet'
+    config = File.join(@target, 'puppet.conf')
+    Puppet.initialize_settings(['--config', config])
+  end
+
+  def run(args=nil)
+    env = Puppet.lookup(:environments).get('benchmarking')
+    node = Puppet::Node.new("testing", :environment => env)
+    Puppet::Resource::Catalog.indirection.find("testing", :use_node => node)
+  end
+
+  def generate
+    environment = File.join(@target, 'environments', 'benchmarking')
+    templates = File.join('benchmarks', 'empty_catalog')
+
+    mkdir_p(File.join(environment, 'modules'))
+    mkdir_p(File.join(environment, 'manifests'))
+
+    render(File.join(templates, 'site.pp.erb'),
+    File.join(environment, 'manifests', 'site.pp'),{})
+
+    render(File.join(templates, 'puppet.conf.erb'),
+           File.join(@target, 'puppet.conf'),
+           :location => @target)
+  end
+
+  def render(erb_file, output_file, bindings)
+    site = ERB.new(File.read(erb_file))
+    File.open(output_file, 'w') do |fh|
+      fh.write(site.result(OpenStruct.new(bindings).instance_eval { binding }))
+    end
+  end
+end

benchmarks/empty_catalog/description

@@ -0,0 +1,4 @@
+Benchmark scenario: an empty catalog (only one call to log a message) shows the setup time for env / compiler
+Benchmark target: catalog compilation overhead
+Parser: Future
+

benchmarks/empty_catalog/puppet.conf.erb

@@ -0,0 +1,5 @@
+confdir = <%= location %>
+vardir = <%= location %>
+environmentpath = <%= File.join(location, 'environments') %>
+environment_timeout = '0'
+parser = future

benchmarks/empty_catalog/site.pp.erb

@@ -0,0 +1 @@
+notice('hello world')