Fix plugin classloading (elastic#14060)

Prior to this commit, attempting to use a custom java plugin installed from rubygems would
fail as the `files` field used to find jar files in the gem is not populated for such gems

This commit uses the `full_require_path` to find the jar file for a native jar plugin,
which should work regardless of where the gem was installed from, locally or via rubygems

Additionally, this commit will only attempt to locate the jar file inside the gem when it
is required (only when the experimental separate plugin class loader is used)

This commit builds on the work done in elastic#13888

Relates: elastic#13888, elastic#11305

Co-authored-by: twosom <[email protected]>
Co-authored-by: Ry Biesemeyer <[email protected]>

logstash-core/lib/logstash/plugins/registry.rb

@@ -152,12 +152,13 @@ def load_available_plugins
 
       GemRegistry.logstash_plugins.each do |plugin_context|
         if plugin_context.spec.metadata.key?('java_plugin')
-          jar_files = plugin_context.spec.files.select {|f| f =~ /.*\.jar/}
+          # Find *.jar file from the require path
+          jar_files = plugin_context.spec.matches_for_glob("**/*.jar")
           expected_jar_name = plugin_context.spec.name + "-" + plugin_context.spec.version.to_s + ".jar"
-          if (jar_files.length != 1 || !jar_files[0].end_with?(expected_jar_name))
+          if jar_files.length != 1 || !jar_files.first.end_with?(expected_jar_name)
             raise LoadError, "Java plugin '#{plugin_context.spec.name}' does not contain a single jar file with the plugin's name and version"
           end
-          @java_plugins[plugin_context.spec.name] = [plugin_context.spec.loaded_from, jar_files[0]]
+          @java_plugins[plugin_context.spec.name] = jar_files.first
         end
 
         # When a plugin has a HOOK_FILE defined, its the responsibility of the plugin
@@ -317,13 +318,11 @@ def add_plugin(type, name, klass)
           else
             raise LoadError,  "Could not find metadata for Java plugin: #{full_name}"
           end
-
           java_import org.logstash.plugins.PluginClassLoader
           java_import org.logstash.Logstash
 
-          classloader = PluginClassLoader.create(plugin_paths[0], plugin_paths[1], Logstash.java_class.class_loader)
+          classloader = PluginClassLoader.create(plugin_paths, Logstash.java_class.class_loader)
           klazz = classloader.load_class(klass.javaClass.name)
-
           @registry[key_for(type, name)] = PluginSpecification.new(type, name, klazz.ruby_class.java_class)
         else
           @registry[key_for(type, name)] = PluginSpecification.new(type, name, klass.javaClass)

logstash-core/src/main/java/org/logstash/plugins/PluginClassLoader.java

@@ -44,18 +44,15 @@ private PluginClassLoader(URL[] urls, ClassLoader appClassLoader) {
 
     /**
      * Creates an instance of the plugin classloader.
-     * @param gemPath Path to the Ruby gem containing the Java plugin as reported by
-     *                <code>Gem::BasicSpecification#loaded_from</code>.
-     * @param jarPath Path to the Java plugin's JAR file relative to {@code gemPath}.
+     * @param jarPath Full path to the Java plugin's JAR file.
      * @param appClassLoader Application classloader to be used for classes not found
      *                       in the plugin's JAR file.
      * @return New instance of the plugin classloader.
      */
-    public static PluginClassLoader create(String gemPath, String jarPath, ClassLoader appClassLoader) {
-        String pluginPath = gemPath.substring(0, gemPath.lastIndexOf(File.separator)) + File.separator + jarPath;
-        Path pluginJar = Paths.get(pluginPath);
+    public static PluginClassLoader create(String jarPath, ClassLoader appClassLoader) {
+        Path pluginJar = Paths.get(jarPath);
         if (!Files.exists(pluginJar)) {
-            throw new IllegalStateException("PluginClassLoader unable to locate jar file: " + pluginPath);
+            throw new IllegalStateException("PluginClassLoader unable to locate jar file: " + jarPath);
         }
         try {
             URL[] pluginJarUrl = new URL[]{pluginJar.toUri().toURL()};

qa/integration/fixtures/install_java_plugin_spec.yml

@@ -0,0 +1,13 @@
+---
+services:
+  - logstash
+config: |-
+ input {
+    java_input_example {
+      count => 4
+    }
+    stdin{}
+ }
+ output {
+   null {}
+ }

qa/integration/specs/install_java_plugin_spec.rb

@@ -0,0 +1,80 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require_relative '../framework/fixture'
+require_relative '../framework/settings'
+require_relative '../framework/helpers'
+require "logstash/devutils/rspec/spec_helper"
+require "stud/temporary"
+
+describe "Install and run java plugin" do
+
+  before(:all) do
+    @fixture = Fixture.new(__FILE__)
+    @logstash = @fixture.get_service("logstash")
+    @logstash_plugin = @logstash.plugin_cli
+  end
+
+  after(:all) {
+    @logstash.teardown
+    @fixture.teardown
+  }
+
+  after(:each) {
+    # cleanly remove the installed plugin to don't pollute
+    # the environment for other subsequent tests
+    removal = @logstash_plugin.run_raw("bin/logstash-plugin uninstall #{plugin_name}")
+
+    expect(removal.stderr_and_stdout).to match(/Successfully removed #{plugin_name}/)
+    expect(removal.exit_code).to eq(0)
+  }
+
+  let(:max_retry) { 120 }
+  let!(:settings_dir) { Stud::Temporary.directory }
+  let(:plugin_name) { "logstash-input-java_input_example" }
+  let(:install_command) { "bin/logstash-plugin install" }
+
+  it "successfully runs a pipeline with an installed Java plugins" do
+    execute = @logstash_plugin.run_raw("#{install_command} #{plugin_name}")
+
+    expect(execute.stderr_and_stdout).to match(/Installation successful/)
+    expect(execute.exit_code).to eq(0)
+
+    installed = @logstash_plugin.list(plugin_name)
+    expect(installed.stderr_and_stdout).to match(/#{plugin_name}/)
+
+    @logstash.start_background_with_config_settings(config_to_temp_file(@fixture.config), settings_dir)
+
+    # wait for Logstash to start
+    started = false
+    while !started
+      begin
+        sleep(1)
+        result = @logstash.monitoring_api.event_stats
+        started = !result.nil?
+      rescue
+        retry
+      end
+    end
+
+    Stud.try(max_retry.times, RSpec::Expectations::ExpectationNotMetError) do
+      result = @logstash.monitoring_api.event_stats
+      expect(result["in"]).to eq(4)
+    end
+
+  end
+end