Introduce the concept of a single spec for the tests, we removed the …

…platform depant test as they where mostly similar, but only used with small platform dependant metadata, this all has been moved to factory methods. make sure platforms uses canonicals "former" names, so no "ubuntu" platfrom as is debian based, and no "centos" as is redhat based.

Also:

* Refactored the specs organization to make the resoning behind simplier,
introducing the idea of an artifact subject that it wraps the
interactions with the platform.

* Add methods to destroy, bootstrap and halt a list of machines either all of them or the ones listed under a given platform name.

* Introduced more clear namespacing in the rakefile.

* Updated the list of available platforms for acceptance testing, including latest ubuntu, oel, fedora, debian and suse versions

Fixes elastic#5350

ci/ci_acceptance.sh

@@ -8,17 +8,18 @@ export JRUBY_OPTS="-J-Xmx1g"
 
 SELECTED_TEST_SUITE=$1
 
-if [[ $SELECTED_TEST_SUITE == $"centos" ]]; then
+if [[ $SELECTED_TEST_SUITE == $"redhat" ]]; then
   echo "Generating the RPM, make sure you start with a clean environment before generating other packages."
   rake artifact:rpm
   echo "Acceptance: Installing dependencies"
   cd qa
   bundle install
 
   echo "Acceptance: Running the tests"
-  bundle exec rake test:setup
-  bundle exec rake test:ssh_config
-  bundle exec rake test:acceptance:centos
+  bundle exec rake qa:vm:setup["redhat"]
+  bundle exec rake qa:vm:ssh_config
+  bundle exec rake qa:acceptance:redhat
+  bundle exec rake qa:vm:halt["redhat"]
 elif [[ $SELECTED_TEST_SUITE == $"debian" ]]; then
   echo "Generating the DEB, make sure you start with a clean environment before generating other packages."
   rake artifact:deb
@@ -27,9 +28,10 @@ elif [[ $SELECTED_TEST_SUITE == $"debian" ]]; then
   bundle install
 
   echo "Acceptance: Running the tests"
-  bundle exec rake test:setup
-  bundle exec rake test:ssh_config
-  bundle exec rake test:acceptance:debian
+  bundle exec rake qa:vm:setup["debian"]
+  bundle exec rake qa:vm:ssh_config
+  bundle exec rake qa:acceptance:debian
+  bundle exec rake qa:vm:halt["debian"]
 elif [[ $SELECTED_TEST_SUITE == $"all" ]]; then
   echo "Building Logstash artifacts"
   rake artifact:all
@@ -39,8 +41,9 @@ elif [[ $SELECTED_TEST_SUITE == $"all" ]]; then
   bundle install
 
   echo "Acceptance: Running the tests"
-  bundle exec rake test:setup
-  bundle exec rake test:ssh_config
-  bundle exec rake test:acceptance:all
+  bundle exec rake qa:vm:setup
+  bundle exec rake qa:vm:ssh_config
+  bundle exec rake qa:acceptance:all
+  bundle exec rake qa:vm:halt
   cd ..
 fi

qa/Gemfile

@@ -2,3 +2,4 @@ source "https://rubygems.org"
 gem "runner-tool", :git => "https://github.com/purbon/runner-tool.git"
 gem "rspec", "~> 3.1.0"
 gem "rake"
+gem "pry", :group => :test

qa/Rakefile

@@ -9,39 +9,61 @@ platforms = PlatformConfig.new
 task :spec    => 'spec:all'
 task :default => :spec
 
-namespace :test do
-  desc "Generate a valid ssh-config"
-  task :ssh_config do
-    require "json"
-    raw_ssh_config    = LogStash::VagrantHelpers.fetch_config.stdout.split("\n");
-    parsed_ssh_config = LogStash::VagrantHelpers.parse(raw_ssh_config)
-    File.write(File.join(File.dirname(__FILE__), ".vm_ssh_config"), parsed_ssh_config.to_json)
-  end
+namespace :qa do
+
+  namespace :vm do
+
+    desc "Generate a valid ssh-config"
+    task :ssh_config do
+      require "json"
+      raw_ssh_config    = LogStash::VagrantHelpers.fetch_config.stdout.split("\n");
+      parsed_ssh_config = LogStash::VagrantHelpers.parse(raw_ssh_config)
+      File.write(".vm_ssh_config", parsed_ssh_config.to_json)
+    end
+
+    desc "Bootstrap all the VM's used for this tests"
+    task :setup, :platform do |t, args|
+      config   = PlatformConfig.new
+      machines = config.select_names_for(args[:platform])
 
-  desc "Bootstrap all the VM's used for this tests"
-  task "setup" do
-    puts "bootstraping all VM's defined in acceptance/Vagrantfile"
-    LogStash::VagrantHelpers.bootstrap
+      message  = "bootstraping all VM's defined in acceptance/Vagrantfile"
+      message  = "#{message} for #{args[:platform]}: #{machines}" if !args[:platform].nil?
+      puts message
+
+      LogStash::VagrantHelpers.destroy(machines)
+      LogStash::VagrantHelpers.bootstrap(machines)
+    end
+
+    desc "Halt all VM's involved in the acceptance test round"
+    task :halt, :platform do |t, args|
+      config   = PlatformConfig.new
+      machines = config.select_names_for(args[:platform])
+      message = "halting all VM's defined inside Vagrantfile"
+      message  = "#{message} for #{args[:platform]}: #{machines}" if !args[:platform].nil?
+      puts message
+
+      LogStash::VagrantHelpers.halt(machines)
+    end
   end
 
   namespace :acceptance do
     desc "Run all acceptance"
     task :all do
-      exit(RSpec::Core::Runner.run([Rake::FileList["acceptance/spec/**/*_spec.rb"]]))
+      exit(RSpec::Core::Runner.run([Rake::FileList["acceptance/spec/lib/*_spec.rb"]]))
     end
 
     platforms.types.each do |type|
       desc "Run acceptance test in #{type} machines"
       task type do
-        exit(RSpec::Core::Runner.run([Rake::FileList["acceptance/spec/#{type}/**/*_spec.rb"]]))
+        ENV['LS_TEST_PLATFORM']=type
+        exit(RSpec::Core::Runner.run([Rake::FileList["acceptance/spec/lib/*_spec.rb"]]))
       end
     end
 
     desc "Run one single machine acceptance test"
-    task :single, :machine do  |t, args|
-      ENV['LS_VAGRANT_HOST'] = args[:machine]
-      platform = LogStash::VagrantHelpers.translate(args[:machine])
-      exit(RSpec::Core::Runner.run([Rake::FileList["acceptance/spec/**/*_spec.rb"]]))
+    task :single, :machine do |t, args|
+      ENV['LS_VAGRANT_HOST']  = args[:machine]
+      exit(RSpec::Core::Runner.run([Rake::FileList["acceptance/spec/lib/*_spec.rb"]]))
     end
   end
 end

qa/acceptance/spec/config_helper.rb

@@ -3,25 +3,20 @@
 
 module SpecsHelper
 
-  def self.find_selected_boxes(default_boxes=[])
-    if ENV.include?('LS_VAGRANT_HOST') then
-      default_boxes.include?(ENV['LS_VAGRANT_HOST']) ? ENV['LS_VAGRANT_HOST'] : []
-    else
-      default_boxes
-    end
-  end
-
   def self.configure(vagrant_boxes)
     setup_config = JSON.parse(File.read(File.join(File.dirname(__FILE__), "..", "..", ".vm_ssh_config")))
-
+    boxes        = vagrant_boxes.inject({}) do |acc, v|
+      acc[v.name] = v.type
+      acc
+    end
     ServiceTester.configure do |config|
       config.servers = []
       config.lookup  = {}
       setup_config.each do |host_info|
-        next unless vagrant_boxes.include?(host_info["host"])
+        next unless boxes.keys.include?(host_info["host"])
         url = "#{host_info["hostname"]}:#{host_info["port"]}"
         config.servers << url
-        config.lookup[url] = host_info["host"]
+        config.lookup[url] = {"host" => host_info["host"], "type" => boxes[host_info["host"]] }
       end
     end
   end

qa/acceptance/spec/lib/artifact_operation_spec.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+require_relative '../spec_helper'
+require_relative '../shared_examples/installed'
+require_relative '../shared_examples/running'
+
+describe "artifacts operation" do
+  config = ServiceTester.configuration
+  config.servers.each do |address|
+    logstash = ServiceTester::Artifact.new(address, config.lookup[address])
+    it_behaves_like "installable", logstash
+    it_behaves_like "runnable",    logstash
+  end
+end

qa/acceptance/spec/shared_examples/installed.rb

@@ -0,0 +1,24 @@
+require_relative '../spec_helper'
+require          'logstash/version'
+
+RSpec.shared_examples "installable" do |logstash|
+
+  before(:each) do
+    logstash.install(LOGSTASH_VERSION)
+  end
+
+  it "is installed on #{logstash.host}" do
+    expect(logstash).to be_installed
+  end
+
+  it "is running on #{logstash.host}" do
+    logstash.start_service
+    expect(logstash).to be_running
+    logstash.stop_service
+  end
+
+  it "is removable on #{logstash.host}" do
+    logstash.uninstall
+    expect(logstash).to be_removed
+  end
+end

qa/acceptance/spec/shared_examples/running.rb

@@ -0,0 +1,16 @@
+require_relative '../spec_helper'
+require          'logstash/version'
+
+RSpec.shared_examples "runnable" do |logstash|
+
+  before(:each) do
+    logstash.install(LOGSTASH_VERSION)
+  end
+
+  it "is running on #{logstash.host}" do
+    logstash.start_service
+    expect(logstash).to be_running
+    logstash.stop_service
+  end
+
+end

qa/acceptance/spec/spec_helper.rb

@@ -3,6 +3,7 @@
 require_relative '../../rspec/helpers'
 require_relative '../../rspec/matchers'
 require_relative 'config_helper'
+require_relative "../../platform_config"
 
 ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..', '..', '..'))
 $LOAD_PATH.unshift File.join(ROOT, 'logstash-core/lib')
@@ -12,3 +13,18 @@
 RSpec.configure do |c|
   c.include ServiceTester
 end
+
+platform = ENV['LS_TEST_PLATFORM'] || 'all'
+
+config   = PlatformConfig.new
+default_vagrant_boxes = ( platform == 'all' ? config.platforms : config.filter_type(platform) )
+
+selected_boxes = if ENV.include?('LS_VAGRANT_HOST') then
+                   config.platforms.select { |p| p.name  == ENV['LS_VAGRANT_HOST'] }
+                 else
+                   default_vagrant_boxes
+                 end
+
+SpecsHelper.configure(selected_boxes)
+
+puts "[Acceptance specs] running on #{ServiceTester.configuration.hosts}" if !selected_boxes.empty?

qa/platform_config.rb

@@ -2,9 +2,12 @@
 require "json"
 
 class PlatformConfig
+
   Platform = Struct.new(:name, :box, :type)
 
-  DEFAULT_CONFIG_LOCATION = File.join(File.dirname(__FILE__), "platforms.json")
+  DEFAULT_CONFIG_LOCATION = File.join(File.dirname(__FILE__), "platforms.json").freeze
+
+  attr_reader :platforms
 
   def initialize(config_path = DEFAULT_CONFIG_LOCATION)
     @config_path = config_path
@@ -35,6 +38,10 @@ def filter_type(type_name)
     @platforms.select { |platform| platform.type == type_name }
   end
 
+  def select_names_for(platform=nil)
+    !platform.nil? ? filter_type(platform).map{ |p| p.name } : ""
+  end
+
   def types
     @platforms.collect(&:type).uniq.sort
   end

qa/platforms.json

@@ -1,6 +1,13 @@
 { 
   "ubuntu-1204": { "box": "elastic/ubuntu-12.04-x86_64", "type": "debian" },
   "ubuntu-1404": { "box": "elastic/ubuntu-14.04-x86_64", "type": "debian" },
-  "centos-6": { "box": "elastic/centos-6-x86_64", "type": "centos" },
-  "centos-7": { "box": "elastic/centos-7-x86_64", "type": "centos" }
+  "ubuntu-1504": { "box": "elastic/ubuntu-15.04-x86_64", "type": "debian" },
+  "centos-6": { "box": "elastic/centos-6-x86_64", "type": "redhat" },
+  "centos-7": { "box": "elastic/centos-7-x86_64", "type": "redhat" },
+  "oel-6": { "box": "elastic/oraclelinux-6-x86_64", "type": "redhat" },
+  "oel-7": { "box": "elastic/oraclelinux-7-x86_64", "type": "redhat" },
+  "fedora-22": { "box": "elastic/fedora-22-x86_64", "type": "redhat" },
+  "debian-8": { "box": "elastic/debian-8-x86_64", "type": "debian" },
+  "opensuse-13": { "box": "elastic/opensuse-13-x86_64", "type": "suse" },
+  "sles-12": { "box": "elastic/sles-12-x86_64", "type": "suse" }
 }