Feature: A way to install/remove a plugin pack

A pack in this context is a *bundle* of plugins that can be distributed outside of rubygems; it is similar to what ES and kibana are doing, and
the user interface is modeled after them. See https://www.elastic.co/downloads/x-pack

**Do not mix it with the `bin/logstash-plugin pack/unpack` command.**

- it contains one or more plugins that need to be installed
- it is self-contains with the gems and the needed jars
- it is distributed as a zip file
- the file structure needs to follow some rules.

- As a reserved name name on elastic.co download http server
    - `bin/plugin install logstash-mypack` will check on the download server if a pack for the current specific logstash version exist and it will be downloaded, if it doesn't exist we fallback on rubygems.
    - The file on the server will follow this convention `logstash-mypack-{LOGSTASH_VERSION}.zip`

- As a fully qualified url
    - `bin/plugin install http://test.abc/logstash-mypack.zip`, if it exists it will be downloaded and installed if it does not we raise an error.

- As a local file
    - `bin/plugin install file:///tmp/logstash-mypack.zip`, if it exists it will be installed

Fixes elastic#6168

.gitignore

@@ -1,5 +1,6 @@
 .*.swp
-*.gem
+/*.gem
+logstash*/*.gem
 pkg/*.deb
 pkg/*.rpm
 *.class

Gemfile

@@ -6,10 +6,11 @@ gem "logstash-core", :path => "./logstash-core"
 gem "logstash-core-queue-jruby", :path => "./logstash-core-queue-jruby"
 gem "logstash-core-event-java", :path => "./logstash-core-event-java"
 gem "logstash-core-plugin-api", :path => "./logstash-core-plugin-api"
+gem "ruby-progressbar", "~> 1.8.1"
+gem "builder", "~> 3.2.2"
 gem "file-dependencies", "0.1.6"
 gem "ci_reporter_rspec", "1.0.0", :group => :development
 gem "simplecov", :group => :development
-gem "coveralls", :group => :development
 gem "tins", "1.6", :group => :development
 gem "rspec", "~> 3.1.0", :group => :development
 gem "logstash-devutils", "~> 1.1", :group => :development
@@ -115,4 +116,3 @@ gem "logstash-output-stdout"
 gem "logstash-output-tcp"
 gem "logstash-output-udp"
 gem "logstash-output-webhdfs"
-gem "logstash-filter-multiline"

lib/bootstrap/bundler.rb

@@ -103,6 +103,7 @@ def invoke!(options = {})
       ::Bundler.settings[:path] = LogStash::Environment::BUNDLE_DIR
       ::Bundler.settings[:gemfile] = LogStash::Environment::GEMFILE_PATH
       ::Bundler.settings[:without] = options[:without].join(":")
+      ::Bundler.settings[:force] = options[:force]
 
       if !debug?
         # Will deal with transient network errors

lib/bootstrap/environment.rb

@@ -5,6 +5,7 @@
 
 require_relative "bundler"
 require_relative "rubygems"
+require "pathname"
 
 module LogStash
   module Environment
@@ -16,7 +17,9 @@ module Environment
     BUNDLE_DIR = ::File.join(LOGSTASH_HOME, "vendor", "bundle")
     GEMFILE_PATH = ::File.join(LOGSTASH_HOME, "Gemfile")
     LOCAL_GEM_PATH = ::File.join(LOGSTASH_HOME, 'vendor', 'local_gems')
-    CACHE_PATH = File.join(LOGSTASH_HOME, "vendor", "cache")
+    CACHE_PATH = ::File.join(LOGSTASH_HOME, "vendor", "cache")
+    LOCKFILE = Pathname.new(::File.join(LOGSTASH_HOME, "Gemfile.jruby-1.9.lock"))
+    GEMFILE = Pathname.new(::File.join(LOGSTASH_HOME, "Gemfile"))
 
     # @return [String] the ruby version string bundler uses to craft its gem path
     def gem_ruby_version

lib/bootstrap/util/compress.rb

@@ -18,13 +18,13 @@ module Zip
       # @param source [String] The location of the file to extract
       # @param target [String] Where you do want the file to be extracted
       # @raise [IOError] If the target directory already exist
-      def extract(source, target)
+      def extract(source, target, pattern = nil)
         raise CompressError.new("Directory #{target} exist") if ::File.exist?(target)
         ::Zip::File.open(source) do |zip_file|
           zip_file.each do |file|
             path = ::File.join(target, file.name)
             FileUtils.mkdir_p(::File.dirname(path))
-            zip_file.extract(file, path)
+            zip_file.extract(file, path) if pattern.nil? || pattern =~ file.name
           end
         end
       end

lib/pluginmanager/bundler/logstash_injector.rb

@@ -0,0 +1,50 @@
+# encoding: utf-8
+require "bootstrap/environment"
+require "bundler"
+require "bundler/definition"
+require "bundler/dependency"
+require "bundler/dsl"
+require "bundler/injector"
+
+# This class cannot be in the logstash namespace, because of the way the DSL
+# class interact with the other libraries
+module Bundler
+  class LogstashInjector < ::Bundler::Injector
+    def self.inject!(new_deps, options = { :gemfile => LogStash::Environment::GEMFILE, :lockfile => LogStash::Environment::LOCKFILE })
+      gemfile = options.delete(:gemfile)
+      lockfile = options.delete(:lockfile)
+
+      bundler_format = Array(new_deps).collect { |plugin|  ::Bundler::Dependency.new(plugin.name, "=#{plugin.version}")}
+
+      injector = new(bundler_format)
+      injector.inject(gemfile, lockfile)
+    end
+
+
+    # This class is pretty similar to what bundler's injector class is doing
+    # but we only accept a local resolution of the dependencies instead of calling rubygems.
+    # so we removed `definition.resolve_remotely!`
+    def inject(gemfile_path, lockfile_path)
+      if Bundler.settings[:frozen]
+        # ensure the lock and Gemfile are synced
+        Bundler.definition.ensure_equivalent_gemfile_and_lockfile(true)
+        # temporarily remove frozen while we inject
+        frozen = Bundler.settings.delete(:frozen)
+      end
+
+      builder = Dsl.new
+      builder.eval_gemfile(gemfile_path)
+
+      @new_deps -= builder.dependencies
+
+      builder.eval_gemfile("injected gems", new_gem_lines) if @new_deps.any?
+      definition = builder.to_definition(lockfile_path, {})
+      append_to(gemfile_path) if @new_deps.any?
+      definition.lock(lockfile_path)
+
+      return @new_deps
+    ensure
+      Bundler.settings[:frozen] = "1" if frozen
+    end
+  end
+end

lib/pluginmanager/bundler/logstash_uninstall.rb

@@ -0,0 +1,89 @@
+# encoding: utf-8
+require "bootstrap/environment"
+require "bundler"
+require "bundler/definition"
+require "bundler/dependency"
+require "bundler/dsl"
+require "bundler/injector"
+require "pluginmanager/gemfile"
+
+# This class cannot be in the logstash namespace, because of the way the DSL
+# class interact with the other libraries
+module Bundler
+  class LogstashUninstall
+    attr_reader :gemfile_path, :lockfile_path
+
+    def initialize(gemfile_path, lockfile_path)
+      @gemfile_path = gemfile_path
+      @lockfile_path = lockfile_path
+    end
+
+    # To be uninstalled the candidate gems need to be standalone.
+    def dependants_gems(gem_name)
+      builder = Dsl.new
+      builder.eval_gemfile("original gemfile", File.read(gemfile_path))
+      definition = builder.to_definition(lockfile_path, {})
+
+      definition.specs
+        .select { |spec| spec.dependencies.collect(&:name).include?(gem_name) }
+        .collect(&:name).sort.uniq
+    end
+
+    def uninstall!(gem_name)
+      unfreeze_gemfile do
+
+        dependencies_from = dependants_gems(gem_name)
+
+        if dependencies_from.size > 0
+          display_cant_remove_message(gem_name, dependencies_from)
+          false
+        else
+          remove_gem(gem_name)
+          true
+        end
+      end
+    end
+
+    def remove_gem(gem_name)
+      builder = Dsl.new
+      file = File.new(gemfile_path, "r+")
+
+      gemfile = LogStash::Gemfile.new(file).load
+      gemfile.remove(gem_name)
+      builder.eval_gemfile("gemfile to changes", gemfile.generate)
+
+      definition = builder.to_definition(lockfile_path, {})
+      definition.lock(lockfile_path)
+      gemfile.save
+
+      LogStash::PluginManager.ui.info("Successfully removed #{gem_name}")
+    ensure
+      file.close if file
+    end
+
+    def display_cant_remove_message(gem_name, dependencies_from)
+        message =<<-eos
+Failed to remove \"#{gem_name}\" because the following plugins or libraries depend on it:
+
+* #{dependencies_from.join("\n* ")}
+        eos
+        LogStash::PluginManager.ui.info(message)
+    end
+
+    def unfreeze_gemfile
+      if Bundler.settings[:frozen]
+        Bundler.definition.ensure_equivalent_gemfile_and_lockfile(true)
+        frozen = Bundler.settings.delete(:frozen)
+      end
+      yield
+    ensure
+      Bundler.settings[:frozen] = "1" if frozen
+    end
+
+    def self.uninstall!(gem_name, options = { :gemfile => LogStash::Environment::GEMFILE, :lockfile => LogStash::Environment::LOCKFILE })
+      gemfile_path = options[:gemfile]
+      lockfile_path = options[:lockfile]
+      LogstashUninstall.new(gemfile_path, lockfile_path).uninstall!(gem_name)
+    end
+  end
+end

lib/pluginmanager/command.rb

@@ -21,7 +21,6 @@ def display_bundler_output(output)
     end
   end
 
-
   # Each plugin install for a gemfile create a path with a unique id.
   # we must clear what is not currently used in the 
   def remove_unused_locally_installed_gems!

lib/pluginmanager/errors.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+module LogStash module PluginManager
+    class PluginManagerError < StandardError; end
+    class FileNotFoundError < PluginManagerError; end
+    class InvalidPackError < PluginManagerError; end
+    class InstallError < PluginManagerError
+      attr_reader :original_exception
+
+      def initialize(original_exception)
+        @original_exception = original_exception
+      end
+    end
+end end

lib/pluginmanager/gem_installer.rb

@@ -0,0 +1,78 @@
+# encoding: utf-8
+require "pluginmanager/ui"
+require "pathname"
+require "rubygems/package"
+
+module LogStash module PluginManager
+  # Install a physical gem package to the appropriate location inside logstash
+  # - Extract the gem
+  # - Generate the specifications
+  # - Copy the data in the right folders
+  class GemInstaller
+    GEM_HOME = Pathname.new(::File.join(LogStash::Environment::BUNDLE_DIR, "jruby", "1.9"))
+    SPECIFICATIONS_DIR = "specifications"
+    GEMS_DIR = "gems"
+
+    attr_reader :gem_home
+
+    def initialize(gem_file, display_post_install_message = false, gem_home = GEM_HOME)
+      @gem = ::Gem::Package.new(gem_file)
+      @gem_home = Pathname.new(gem_home)
+      @display_post_install_message = display_post_install_message
+    end
+
+    def install
+      create_destination_folders
+      extract_files
+      write_specification
+      display_post_install_message
+    end
+
+    def self.install(gem_file, display_post_install_message = false, gem_home = GEM_HOME)
+      self.new(gem_file, display_post_install_message, gem_home).install
+    end
+
+    private
+    def spec
+      @gem.spec
+    end
+
+    def spec_dir
+      gem_home.join(SPECIFICATIONS_DIR)
+    end
+
+    def spec_file
+      spec_dir.join("#{spec.full_name}.gemspec")
+    end
+
+    def gem_dir
+      gem_home.join(GEMS_DIR, spec.full_name)
+    end
+
+    def extract_files
+      @gem.extract_files gem_dir
+    end
+
+    def write_specification
+      ::File.open(spec_file, 'w') do |file|
+        spec.installed_by_version = ::Gem.rubygems_version
+        file.puts spec.to_ruby_for_cache
+        file.fsync rescue nil # Force writing to disk
+      end
+    end
+
+    def display_post_install_message
+      PluginManager.ui.info(spec.post_install_message) if display_post_install_message?
+    end
+
+    def display_post_install_message?
+      @display_post_install_message && !spec.post_install_message.nil?
+    end
+
+    def create_destination_folders
+      FileUtils.mkdir_p(gem_home)
+      FileUtils.mkdir_p(gem_dir)
+      FileUtils.mkdir_p(spec_dir)
+    end
+  end
+end end

lib/pluginmanager/gemfile.rb

@@ -15,21 +15,24 @@ def initialize(io)
       @gemset = nil
     end
 
-    def load
+    def load(with_backup = true)
       @gemset ||= DSL.parse(@io.read)
-      backup
+      backup if with_backup
       self
     end
 
     def save
       raise(GemfileError, "a Gemfile must first be loaded") unless @gemset
       @io.truncate(0)
       @io.rewind
-      @io.write(HEADER)
-      @io.write(@gemset.to_s)
+      @io.write(generate)
       @io.flush
     end
 
+    def generate
+      "#{HEADER}#{gemset.to_s}"
+    end
+
     def find(name)
       @gemset.find_gem(name)
     end

lib/pluginmanager/install.rb

@@ -1,5 +1,8 @@
 # encoding: utf-8
 require "pluginmanager/command"
+require "pluginmanager/install_strategy_factory"
+require "pluginmanager/ui"
+require "pluginmanager/errors"
 require "jar-dependencies"
 require "jar_install_post_install_hook"
 require "file-dependencies/gem"
@@ -17,6 +20,30 @@ class LogStash::PluginManager::Install < LogStash::PluginManager::Command
   # but the argument parsing does not support it for now so currently if specifying --version only
   # one plugin name can be also specified.
   def execute
+    # This is a special flow for PACK related plugins,
+    # if we dont detect an pack we will just use the normal `Bundle install` Strategy`
+    # this could be refactored into his own strategy
+    begin
+      if strategy = LogStash::PluginManager::InstallStrategyFactory.create(plugins_arg)
+        LogStash::PluginManager.ui.debug("Installing with strategy: #{strategy.class}")
+        strategy.execute
+        return
+      end
+    rescue LogStash::PluginManager::InstallError => e
+      report_exception("An error occured when installing the: #{plugins_args_human}, to have more information about the error add a DEBUG=1 before running the command.", e.original_exception)
+      return
+    rescue LogStash::PluginManager::FileNotFoundError => e
+      report_exception("File not found for: #{plugins_args_human}", e)
+      return
+    rescue LogStash::PluginManager::InvalidPackError => e
+      report_exception("Invalid pack for: #{plugins_args_human}, reason: #{e.message}", e)
+      return
+    rescue => e
+      report_exception("Something went wrong when installing #{plugins_args_human}", e)
+      return
+    end
+
+    # TODO(ph): refactor this into his own strategy
     validate_cli_options!
 
     if local_gems?
@@ -152,4 +179,8 @@ def local_gems?
       signal_usage_error("Mixed source of plugins, you can't mix local `.gem` and remote gems")
     end
   end
+
+  def plugins_args_human
+    plugins_arg.join(", ")
+  end
 end # class Logstash::PluginManager