Allow Logstash to accept a PROXY configuration

Logstash's plugin manager will now follow proxy configuration from the environment.
If you configure `http_proxy` and `https_proxy`, the manager will now use theses information for all the ruby http
connection and will also pass that information down to maven.

Fixes: elastic#6619, elastic#6528

Fixes elastic#6825

.travis.yml

@@ -16,7 +16,7 @@ env:
 before_install:
   # Force bundler 1.12.5 because version 1.13 has issues, see https://github.com/fastlane/fastlane/issues/6065#issuecomment-246044617
   - gem uninstall -i /home/travis/.rvm/gems/jruby-1.7.25@global bundler
-  - gem install bundler -v 1.12.5 --no-rdoc --no-ri --no-document --quiet   
+  - gem install bundler -v 1.12.5 --no-rdoc --no-ri --no-document --quiet
 install:
   - rake test:install-core
 before_script:
@@ -28,4 +28,4 @@ script:
         ci/travis_integration_run.sh;
       else
         rake test:core
-      fi      
+      fi

ci/travis_integration_install.sh

@@ -7,12 +7,12 @@ set -e
 if [[ "$INTEGRATION" != "true" ]]; then
     exit
 fi
-  
+
 echo "Setting up integration tests"
 if [[ ! -d "build" ]]; then
     mkdir build
-fi  
-rm -rf build/*  
+fi
+rm -rf build/*
 echo "Building logstash tar file in build/"
 rake artifact:tar
 cd build
@@ -23,4 +23,4 @@ cd ../qa/integration
 pwd
 echo $BUNDLE_GEMFILE
 # to install test dependencies
-bundle install --gemfile="Gemfile"
+bundle install --gemfile="./Gemfile"

docs/static/plugin-manager.asciidoc

@@ -6,6 +6,18 @@ packages called gems and hosted on RubyGems.org. The plugin manager accessed via
 lifecycle of plugins in your Logstash deployment. You can install, remove and upgrade plugins using the Command Line
 Interface (CLI) invocations described below.
 
+[[http-proxy]]
+=== Proxy configuration
+
+The majority of the plugin manager commands require access to the internet to reach https://rubygems.org[RubyGems.org].
+If your organization is behind a firewall you can set these environments variables to configure Logstash to use your proxy.
+
+[source, shell]
+----------------------------------
+export http_proxy=http://localhost:3128
+export https_proxy=http://localhost:3128
+----------------------------------
+
 [float]
 [[listing-plugins]]
 === Listing plugins

lib/pluginmanager/main.rb

@@ -22,6 +22,8 @@ module PluginManager
 require "pluginmanager/unpack"
 require "pluginmanager/generate"
 require "pluginmanager/prepare_offline_pack"
+require "pluginmanager/proxy_support"
+configure_proxy
 
 module LogStash
   module PluginManager

lib/pluginmanager/proxy_support.rb

@@ -0,0 +1,89 @@
+# encoding: utf-8
+require "uri"
+require "java"
+require "erb"
+require "ostruct"
+require "fileutils"
+require "stud/temporary"
+require "jar-dependencies"
+
+
+# This is a bit of a hack, to make sure that all of our call pass to a specific proxies.
+# We do this before any jar-dependences check is done, meaning we have to silence him.
+module Jars
+  def self.warn(message)
+    if ENV["debug"]
+      puts message
+    end
+  end
+end
+
+SETTINGS_TEMPLATE = ::File.join(::File.dirname(__FILE__), "settings.xml.erb")
+SETTINGS_TARGET = ::File.join(Dir.home, ".m2")
+
+class ProxyTemplateData
+  attr_reader :proxies
+
+  def initialize(proxies)
+    @proxies = proxies.collect { |proxy| OpenStruct.new(proxy) }
+  end
+
+  def get_binding
+    binding
+  end
+end
+
+# Apply HTTP_PROXY and HTTPS_PROXY to the current environment
+# this will be used by any JRUBY calls
+def apply_env_proxy_settings(settings)
+  scheme = settings[:protocol].downcase
+  java.lang.System.setProperty("#{scheme}.proxyHost", settings[:host])
+  java.lang.System.setProperty("#{scheme}.proxyPort", settings[:port].to_s)
+  java.lang.System.setProperty("#{scheme}.proxyUsername", settings[:username].to_s)
+  java.lang.System.setProperty("#{scheme}.proxyPassword", settings[:password].to_s)
+end
+
+def extract_proxy_values_from_uri(proxy_uri)
+  proxy_uri = URI(proxy_uri)
+  {
+    :protocol => proxy_uri.scheme,
+    :host => proxy_uri.host,
+    :port => proxy_uri.port,
+    :username => proxy_uri.user,
+    :password => proxy_uri.password
+  }
+end
+
+def configure_proxy
+  proxies = []
+  if proxy = (ENV["http_proxy"] || ENV["HTTP_PROXY"])
+    proxy_settings = extract_proxy_values_from_uri(proxy)
+    proxy_settings[:protocol] = "http"
+    apply_env_proxy_settings(proxy_settings)
+    proxies << proxy_settings
+  end
+
+  if proxy = (ENV["https_proxy"] || ENV["HTTPS_PROXY"])
+    proxy_settings = extract_proxy_values_from_uri(proxy)
+    proxy_settings[:protocol] = "https"
+    apply_env_proxy_settings(proxy_settings)
+    proxies << proxy_settings
+  end
+
+  # I've tried overriding jar dependency environment variable to declare the settings but it doesn't seems to work.
+  # I am not sure if its because of our current setup or its a bug in the library.
+  if !proxies.empty?
+    FileUtils.mkdir_p(SETTINGS_TARGET)
+    target = ::File.join(SETTINGS_TARGET, "settings.xml")
+    template = ::File.read(SETTINGS_TEMPLATE)
+    template_content = ERB.new(template, 3).result(ProxyTemplateData.new(proxies).get_binding)
+
+    if ::File.exist?(target)
+      if template_content != ::File.read(target)
+        puts "WARNING: A maven settings file already exist at #{target}, please review the content to make sure it include your proxies configuration."
+      end
+    else
+      ::File.open(target, "w") { |f| f.write(template_content) }
+    end
+  end
+end

lib/pluginmanager/settings.xml.erb

@@ -0,0 +1,18 @@
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
+  https://maven.apache.org/xsd/settings-1.0.0.xsd">
+  <proxies>
+    <% proxies.each_with_index do |proxy, idx| %>
+    <proxy>
+      <id><%=proxy.host%>.<%=idx%></id>
+      <active>true</active>
+      <protocol><%=proxy.protocol%></protocol>
+      <host><%=proxy.host%></host>
+      <port><%=proxy.port%></port>
+      <username><%=proxy.username%></username>
+      <password><%=proxy.password%></password>
+    </proxy>
+  <% end %>
+  </proxies>
+</settings>

lib/pluginmanager/utils/http_client.rb

@@ -10,7 +10,15 @@ class RedirectionLimit < RuntimeError; end
     # https://ruby-doc.org/stdlib-2.3.1/libdoc/net/http/rdoc/Net/HTTP.html#class-Net::HTTP-label-Proxies
     def self.start(uri)
       uri = URI(uri)
-      Net::HTTP.start(uri.host, uri.port, :use_ssl => uri.scheme == HTTPS_SCHEME) { |http| yield http }
+      Net::HTTP.start(uri.host, uri.port, http_options(uri)) { |http| yield http }
+    end
+
+    def self.http_options(uri)
+      ssl_enabled = uri.scheme == HTTPS_SCHEME
+
+      {
+        :use_ssl => ssl_enabled
+      }
     end
 
     # Do a HEAD request on the file to see if it exist before downloading it

qa/integration/fixtures/http_proxy_install_spec.yml

@@ -0,0 +1,3 @@
+---
+services:
+  - logstash

qa/integration/services/http_proxy_service.rb

@@ -0,0 +1,6 @@
+# encoding: utf-8
+class Http_proxyService < Service
+  def initialize(settings)
+    super("http_proxy", settings)
+  end
+end

qa/integration/services/http_proxy_setup.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -ex
+sudo apt-get install -y squid3 net-tools
+sudo iptables -A OUTPUT -p tcp --dport 80 -m owner --uid-owner proxy -j ACCEPT
+sudo iptables -A OUTPUT -p tcp --dport 443 -m owner --uid-owner proxy -j ACCEPT
+sudo iptables -A OUTPUT -p tcp --dport 80 -m owner --uid-owner root -j ACCEPT
+sudo iptables -A OUTPUT -p tcp --dport 443 -m owner --uid-owner root -j ACCEPT
+sudo iptables -A OUTPUT -p tcp --dport 443 -j REJECT
+sudo iptables -A OUTPUT -p tcp --dport 80 -j REJECT
+
+echo "Connecting to a remote host should fails without proxy"
+curl -I --silent "http://rubygems.org" || echo "Success"
+
+echo "Connecting to a remote host with a valid proxy should succeed"
+export http_proxy=http://localhost:3128
+export https_proxy=http://localhost:3128
+export HTTP_PROXY=http://localhost:3128
+export HTTPS_PROXY=http://localhost:3128
+curl -I --silent "https://rubygems.org" || echo "Success"
+
+echo "Unset the default variables"
+unset http_proxy
+unset https_proxy
+unset HTTP_PROXY
+unset HTTPS_PROXY

qa/integration/services/http_proxy_teardown.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -ex
+
+echo "Removing all the chain"
+sudo iptables -F OUTPUT

qa/integration/services/logstash_service.rb

@@ -238,14 +238,17 @@ def install(plugin_name)
       run("install #{plugin_name}")
     end
 
-    def run_raw(cmd_parameters, change_dir = true)
+    def run_raw(cmd_parameters, change_dir = true, environment = {})
       out = Tempfile.new("content")
       out.sync = true
 
       parts = cmd_parameters.split(" ")
       cmd = parts.shift
 
       process = ChildProcess.build(cmd, *parts)
+      environment.each do |k, v|
+        process.environment[k] = v
+      end
       process.io.stdout = process.io.stderr = out
 
       Bundler.with_clean_env do

qa/integration/specs/cli/http_proxy_install_spec.rb

@@ -0,0 +1,65 @@
+# encoding: utf-8
+require_relative "../../framework/fixture"
+require_relative "../../framework/settings"
+require_relative "../../services/logstash_service"
+require_relative "../../services/http_proxy_service"
+require_relative "../../framework/helpers"
+require "logstash/devutils/rspec/spec_helper"
+require "stud/temporary"
+require "fileutils"
+
+# Theses tests doesn't currently work on Travis, since we need to run them in a sudo
+# environment and we do that other tests are faillings. This is probably due to IPv4 vs IPv6 settings
+# in the VM vs the container.
+#
+# We are working to bring the test to our internal Jenkins environment.
+#
+# describe "(HTTP_PROXY) CLI > logstash-plugin install", :linux => true do
+#   before :all do
+#     @fixture = Fixture.new(__FILE__)
+#     @logstash_cli = @fixture.get_service("logstash").plugin_cli
+#     @http_proxy = @fixture.get_service("http_proxy")
+#   end
+
+#   before(:all) { @http_proxy.setup }
+#   after(:all) { @http_proxy.teardown }
+
+#   before do
+#     # Make sure we don't have any settings from a previous execution
+#     FileUtils.rm_rf(File.join(Dir.home, ".m2", "settings.xml"))
+#     FileUtils.rm_rf(File.join(Dir.home, ".m2", "repository"))
+#   end
+
+#   context "when installing plugins in an airgap environment" do
+#     context "when a proxy is not configured" do
+#       it "should fail" do
+#         environment = {
+#           "http_proxy" => nil,
+#           "https_proxy" => nil,
+#           "HTTP_PROXY" => nil,
+#           "HTTPS_PROXY" => nil,
+#         }
+
+#         execute = @logstash_cli.run_raw(cmd, true, environment)
+
+#         expect(execute.stderr_and_stdout).not_to match(/Installation successful/)
+#         expect(execute.exit_code).to eq(1)
+#       end
+#     end
+
+#     context "when a proxy is configured" do
+#       it "should allow me to install a plugin" do
+#         environment = {
+#           "http_proxy" => "http://localhost:3128",
+#           "https_proxy" => "http://localhost:3128"
+#         }
+
+#         cmd = "bin/logstash-plugin install --no-verify"
+#         execute = @logstash_cli.run_raw(cmd, true, environment)
+
+#         expect(execute.stderr_and_stdout).to match(/Installation successful/)
+#         expect(execute.exit_code).to eq(0)
+#       end
+#     end
+#   end
+# end

qa/integration/specs/spec_helper.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+RSpec.configure do |config|
+  if RbConfig::CONFIG["host_os"] != "linux"
+    exclude_tags = { :linux => true }
+  end
+
+  config.filter_run_excluding exclude_tags
+end