Update jruby version to 9.3.4.0 (elastic#14114)

This commit updates the version of jruby used in Logstash to `9.3.4.0`.

* Updates the references of `jruby` from `9.2.20.1` to `9.3.4.0`
* Updates references/locations of ruby from `2.5.0` to `2.6.0`
* Updates java imports including `org.logstash.util` to be quoted
  * Without quoting the name of the import, the following error is observed in tests:
  * `java.lang.NoClassDefFoundError: org/logstash/Util (wrong name: org/logstash/util)`
  * Maybe an instance of jruby/jruby#4861
* Adds a monkey patch to `require` to resolve compatibility issue between latest `jruby` and `polyglot` gem 
  * The addition of jruby/jruby#7145 to disallow circular
causes, will throw when `polyglot` is thrown into the mix, and stop logstash from
starting and building - any gems that use an exception to determine whether or not
to load the native gem, will trigger the code added in that commit.
  * This commit adds a monkey patch of `require` to rollback the circular cause exception
back to the original cause.
* Removes the use of the deprecated `JavaClass`
* Adds additional `require time` in `generate_build_metadata`
* Rewrites a test helper to avoid potentially calling `~>` on `FalseClass`


Co-authored-by: Joao Duarte <[email protected]>
Co-authored-by: João Duarte <[email protected]>

Dockerfile

@@ -35,7 +35,7 @@ ADD LICENSE.txt /opt/logstash/LICENSE.txt
 ADD NOTICE.TXT /opt/logstash/NOTICE.TXT
 ADD licenses /opt/logstash/licenses
 ADD CONTRIBUTORS /opt/logstash/CONTRIBUTORS
-ADD Gemfile.template Gemfile.jruby-2.5.lock.* /opt/logstash/
+ADD Gemfile.template Gemfile.jruby-2.6.lock.* /opt/logstash/
 ADD Rakefile /opt/logstash/Rakefile
 ADD build.gradle /opt/logstash/build.gradle
 ADD rubyUtils.gradle /opt/logstash/rubyUtils.gradle

bin/logstash.lib.sh

@@ -163,7 +163,7 @@ setup_vendored_jruby() {
   fi
 
   if [ -z "$LS_GEM_HOME" ] ; then
-    export GEM_HOME="${LOGSTASH_HOME}/vendor/bundle/jruby/2.5.0"
+    export GEM_HOME="${LOGSTASH_HOME}/vendor/bundle/jruby/2.6.0"
   else
     export GEM_HOME=${LS_GEM_HOME}
   fi

build.gradle

@@ -318,9 +318,9 @@ def bundlerVersion = "~> 2"
 
 tasks.register("installBundler") {
     dependsOn assemblyDeps
-    outputs.files file("${projectDir}/vendor/bundle/jruby/2.5.0/bin/bundle")
+    outputs.files file("${projectDir}/vendor/bundle/jruby/2.6.0/bin/bundle")
     doLast {
-      gem(projectDir, buildDir, "bundler", bundlerVersion, "${projectDir}/vendor/bundle/jruby/2.5.0")
+      gem(projectDir, buildDir, "bundler", bundlerVersion, "${projectDir}/vendor/bundle/jruby/2.6.0")
   }
 }
 
@@ -334,9 +334,9 @@ tasks.register("bootstrap"){
 
 tasks.register("installDefaultGems") {
     dependsOn bootstrap
-  doLast {
-      rake(projectDir, buildDir, 'plugin:install-default')
-  }
+    doLast {
+        rake(projectDir, buildDir, 'plugin:install-default')
+    }
 }
 
 tasks.register("installTestGems") {
@@ -441,7 +441,7 @@ tasks.register("unpackTarDistribution", Copy) {
 
 def qaBuildPath = "${buildDir}/qa/integration"
 def qaVendorPath = "${qaBuildPath}/vendor"
-def qaBundledGemPath = "${qaVendorPath}/jruby/2.5.0".toString()
+def qaBundledGemPath = "${qaVendorPath}/jruby/2.6.0".toString()
 def qaBundleBin = "${qaBundledGemPath}/bin/bundle"
 
 tasks.register("installIntegrationTestBundler"){

lib/bootstrap/bundler.rb

@@ -119,7 +119,7 @@ def invoke!(options = {})
         )
       end
       # create Gemfile.jruby-1.9.lock from template iff a template exists it itself does not exist
-      lock_template = ::File.join(ENV["LOGSTASH_HOME"], "Gemfile.jruby-2.5.lock.release")
+      lock_template = ::File.join(ENV["LOGSTASH_HOME"], "Gemfile.jruby-2.6.lock.release")
       if ::File.exists?(lock_template) && !::File.exists?(Environment::LOCKFILE)
         FileUtils.copy(lock_template, Environment::LOCKFILE)
       end

lib/pluginmanager/gem_installer.rb

@@ -26,7 +26,7 @@ module LogStash module PluginManager
   # - Generate the specifications
   # - Copy the data in the right folders
   class GemInstaller
-    GEM_HOME = Pathname.new(::File.join(LogStash::Environment::BUNDLE_DIR, "jruby", "2.5.0"))
+    GEM_HOME = Pathname.new(::File.join(LogStash::Environment::BUNDLE_DIR, "jruby", "2.6.0"))
     SPECIFICATIONS_DIR = "specifications"
     GEMS_DIR = "gems"
     CACHE_DIR = "cache"

lib/pluginmanager/util.rb

@@ -68,9 +68,9 @@ def self.logstash_plugin?(plugin, version = nil, options={})
         return false
       end
     else
-      dep = Gem::Dependency.new(plugin, version || Gem::Requirement.default)
-      Gem.sources = Gem::SourceList.from(options[:rubygems_source]) if options[:rubygems_source]
-      specs, errors = Gem::SpecFetcher.fetcher.spec_for_dependency(dep)
+      dep = ::Gem::Dependency.new(plugin, version || ::Gem::Requirement.default)
+      ::Gem.sources = ::Gem::SourceList.from(options[:rubygems_source]) if options[:rubygems_source]
+      specs, errors = ::Gem::SpecFetcher.fetcher.spec_for_dependency(dep)
 
       # dump errors
       errors.each { |error| $stderr.puts(error.wordy) }
@@ -112,7 +112,7 @@ def self.logstash_plugin_gem_spec?(spec)
   # @return [Gem::Specification] .get file gem specification
   # @raise [Exception] Gem::Package::FormatError will be raised on invalid .gem file format, might be other exceptions too
   def self.plugin_file_spec(path)
-    Gem::Package.new(path).spec
+    ::Gem::Package.new(path).spec
   end
 
   # @param plugin [String] the plugin name or the local path to a .gem file
@@ -125,7 +125,7 @@ def self.plugin_file?(plugin)
   # @param name [String] specific plugin name to find or nil for all plugins
   # @return [Array<Gem::Specification>] all local logstash plugin gem specs
   def self.find_plugins_gem_specs(name = nil)
-    specs = name ? Gem::Specification.find_all_by_name(name) : Gem::Specification.find_all
+    specs = name ? ::Gem::Specification.find_all_by_name(name) : ::Gem::Specification.find_all
     specs.select{|spec| logstash_plugin_gem_spec?(spec)}
   end
 

logstash-core/lib/logstash/api/app_helpers.rb

@@ -60,8 +60,11 @@ def extract_fields(filter_string)
   end
 
   def as_boolean(string)
-    return true   if string == true   || string =~ (/(true|t|yes|y|1)$/i)
-    return false  if string == false  || LogStash::Util.blank?(string) || string =~ (/(false|f|no|n|0)$/i)
+    return true if string == true
+    return false if string == false
+
+    return true if string =~ (/(true|t|yes|y|1)$/i)
+    return false if  LogStash::Util.blank?(string) || string =~ (/(false|f|no|n|0)$/i)
     raise ArgumentError.new("invalid value for Boolean: \"#{string}\"")
   end
 

logstash-core/lib/logstash/compiler/lscl.rb

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+require "logstash/patches/polyglot"
 require "treetop"
 require "logstash/compiler/treetop_monkeypatches"
 require "logstash/compiler/lscl/helpers"
@@ -112,7 +113,7 @@ def plugin_name
     def expr_attributes
       # Turn attributes into a hash map
       self.attributes.recursive_select(Attribute).map(&:expr).map {|k,v|
-        if v.java_kind_of?(Java::OrgLogstashConfigIrExpression::ValueExpression)
+        if v.kind_of?(Java::OrgLogstashConfigIrExpression::ValueExpression)
           [k, v.get]
         else
           [k,v]
@@ -347,7 +348,7 @@ def precedence(op)
     def jconvert(sexpr)
       raise "jconvert cannot handle nils!" if sexpr.nil?
 
-      if sexpr.java_kind_of?(Java::OrgLogstashConfigIrExpression::Expression)
+      if sexpr.kind_of?(Java::OrgLogstashConfigIrExpression::Expression)
         return sexpr
       end
 

logstash-core/lib/logstash/config/lir_serializer.rb

@@ -71,13 +71,13 @@ def vertex(v)
     end
 
     def vertex_type(v)
-      if v.java_kind_of?(org.logstash.config.ir.graph.PluginVertex)
+      if v.kind_of?(org.logstash.config.ir.graph.PluginVertex)
         :plugin
-      elsif v.java_kind_of?(org.logstash.config.ir.graph.IfVertex)
+      elsif v.kind_of?(org.logstash.config.ir.graph.IfVertex)
         :if
-      elsif v.java_kind_of?(org.logstash.config.ir.graph.QueueVertex)
+      elsif v.kind_of?(org.logstash.config.ir.graph.QueueVertex)
         :queue
-      elsif v.java_kind_of?(org.logstash.config.ir.graph.SeparatorVertex)
+      elsif v.kind_of?(org.logstash.config.ir.graph.SeparatorVertex)
         :separator
       else
         raise "Unexpected vertex type! #{v}"
@@ -121,7 +121,7 @@ def remove_separators_from_edges(edges)
       edges.each do |e|
         if vertex_type(e.to) == :separator
           e.to.getOutgoingEdges.each do |outgoing|
-            if e.java_kind_of?(org.logstash.config.ir.graph.BooleanEdge)
+            if e.kind_of?(org.logstash.config.ir.graph.BooleanEdge)
               edges_with_separators_removed << edge(org.logstash.config.ir.graph.BooleanEdge.new(e.edgeType, e.from, outgoing.to))
             else
               edges_with_separators_removed << edge(org.logstash.config.ir.graph.PlainEdge.factory.make(e.from, outgoing.to))
@@ -143,7 +143,7 @@ def edge(e)
         "id" => e.id
       }
 
-      if e.java_kind_of?(org.logstash.config.ir.graph.BooleanEdge)
+      if e.kind_of?(org.logstash.config.ir.graph.BooleanEdge)
         e_json["when"] = e.edge_type
         e_json["type"] = "boolean"
       else

logstash-core/lib/logstash/patches/polyglot.rb

@@ -0,0 +1,19 @@
+require 'polyglot'
+
+
+module Kernel
+  alias original_require require
+
+  def require(*a, &b)
+    begin
+      original_require(*a, &b)
+    rescue RuntimeError => e
+      # https://github.com/jruby/jruby/pull/7145 introduced an exception check for circular causes, which
+      # breaks when the polyglot library is used and LoadErrors are emitted
+      if e.message == "circular causes"
+        raise e.cause
+      end
+      raise e
+    end
+  end
+end

logstash-core/lib/logstash/plugins/registry.rb

@@ -293,7 +293,6 @@ def namespace_lookup(type, name)
     # @return [Boolean] true if klass is a valid plugin for name
     def is_a_plugin?(klass, name)
       (klass.class == Java::JavaLang::Class && klass.simple_name.downcase == name.gsub('_','')) ||
-      (klass.class == Java::JavaClass && klass.simple_name.downcase == name.gsub('_','')) ||
       (klass.ancestors.include?(LogStash::Plugin) && klass.respond_to?(:config_name) &&
         klass.config_name == name)
     end

logstash-core/lib/logstash/runner.rb

@@ -32,7 +32,6 @@
 end
 
 require "clamp"
-
 require "logstash-core/logstash-core"
 require "logstash/environment"
 require "logstash/modules/cli_parser"

logstash-core/lib/logstash/util/cloud_setting_auth.rb

@@ -16,5 +16,5 @@
 # under the License.
 
 module LogStash; module Util
-  java_import org.logstash.util.CloudSettingAuth
+  java_import "org.logstash.util.CloudSettingAuth"
 end; end

logstash-core/lib/logstash/util/cloud_setting_id.rb

@@ -18,5 +18,5 @@
 require "base64"
 
 module LogStash; module Util
-    java_import org.logstash.util.CloudSettingId
+    java_import "org.logstash.util.CloudSettingId"
 end; end

logstash-core/lib/logstash/util/modules_setting_array.rb

@@ -16,5 +16,5 @@
 # under the License.
 
 module LogStash; module Util
-    java_import org.logstash.util.ModulesSettingArray
+    java_import "org.logstash.util.ModulesSettingArray"
 end; end

logstash-core/lib/logstash/util/time_value.rb

@@ -17,6 +17,6 @@
 
 module LogStash
   module Util
-    java_import org.logstash.util.TimeValue
+    java_import "org.logstash.util.TimeValue"
   end
 end

logstash-core/spec/logstash/util_spec.rb

@@ -90,17 +90,5 @@ class TestKlass
       expect( thread_id ).to be_a Integer
       expect( thread_id ).to eq(java.lang.Thread.currentThread.getId)
     end
-
-    context "when a (native) thread is collected" do
-      let(:dead_thread) { Thread.new { 42 }.tap { |t| sleep(0.01) while t.status } }
-
-      it "returns nil as id" do
-        thread = dead_thread
-        p thread if $VERBOSE
-        2.times { java.lang.System.gc || sleep(0.01) } # we're assuming a full-gc to clear all weak-refs
-        # NOTE: if you notice this spec failing - remote it (a java.lang.Thread weak-ref might stick around)
-        expect(LogStash::Util.get_thread_id(thread)).to be nil
-      end
-    end
   end
 end

logstash-core/spec/support/helpers.rb

@@ -110,7 +110,7 @@ def temporary_file(content, file_name = Time.now.to_i.to_s, directory = Stud::Te
 
 RSpec::Matchers.define :ir_eql do |expected|
   match do |actual|
-    next unless expected.java_kind_of?(org.logstash.config.ir.SourceComponent) && actual.java_kind_of?(org.logstash.config.ir.SourceComponent)
+    next unless expected.kind_of?(org.logstash.config.ir.SourceComponent) && actual.kind_of?(org.logstash.config.ir.SourceComponent)
 
     expected.sourceComponentEquals(actual)
   end

logstash-core/src/main/java/org/logstash/RubyJavaIntegration.java

@@ -31,9 +31,11 @@
 import org.jruby.Ruby;
 import org.jruby.RubyBasicObject;
 import org.jruby.RubyString;
+import org.jruby.RubyModule;
 import org.jruby.anno.JRubyMethod;
 import org.jruby.java.proxies.JavaProxy;
 import org.jruby.java.proxies.MapJavaProxy;
+import org.jruby.javasupport.Java;
 import org.jruby.javasupport.JavaClass;
 import org.jruby.javasupport.JavaUtil;
 import org.jruby.runtime.Block;
@@ -60,14 +62,14 @@ public static void setupRubyJavaIntegration(final Ruby ruby) {
         ruby.getArray().defineAnnotatedMethods(RubyJavaIntegration.RubyArrayOverride.class);
         ruby.getHash().defineAnnotatedMethods(RubyJavaIntegration.RubyHashOverride.class);
         Stream.of(LinkedHashMap.class, HashMap.class).forEach(cls ->
-            JavaClass.get(ruby, cls).getProxyModule().defineAnnotatedMethods(
+            Java.getProxyClass(ruby, cls).defineAnnotatedMethods(
                 RubyJavaIntegration.RubyMapProxyOverride.class
             )
         );
-        JavaClass.get(ruby, Map.class).getProxyModule().defineAnnotatedMethods(
+        Java.getProxyClass(ruby, Map.class).defineAnnotatedMethods(
             RubyJavaIntegration.JavaMapOverride.class
         );
-        JavaClass.get(ruby, Collection.class).getProxyModule().defineAnnotatedMethods(
+        Java.getProxyClass(ruby, Collection.class).defineAnnotatedMethods(
             RubyJavaIntegration.JavaCollectionOverride.class
         );
     }
@@ -279,7 +281,8 @@ public static IRubyObject containsKey(final ThreadContext context, final IRubyOb
         @JRubyMethod
         public static IRubyObject merge(final ThreadContext context, final IRubyObject self,
             final IRubyObject other) {
-            return ((MapJavaProxy) self.dup()).merge_bang(context, other, Block.NULL_BLOCK);
+            IRubyObject[] other_array = { other };
+            return ((MapJavaProxy) self.dup()).merge_bang(context, other_array, Block.NULL_BLOCK);
         }
     }
 }

logstash-core/src/test/java/org/logstash/config/ir/RubyEnvTestCase.java

@@ -23,6 +23,7 @@
 import java.nio.file.Path;
 
 import org.jruby.RubyHash;
+import org.jruby.runtime.load.LibrarySearcher;
 import org.jruby.runtime.load.LoadService;
 import org.junit.BeforeClass;
 import org.logstash.RubyTestBase;
@@ -41,9 +42,10 @@ public static void before() {
      */
     private static void ensureLoadpath() {
         final LoadService loader = RubyUtil.RUBY.getLoadService();
-        if (loader.findFileForLoad("logstash/compiler").library == null) {
+        final LibrarySearcher librarySearcher = new LibrarySearcher(loader);
+        if (librarySearcher.findLibraryForLoad("logstash/compiler") == null) {
             final String gems = LS_HOME.
-                    resolve("vendor").resolve("bundle").resolve("jruby").resolve("2.5.0").
+                    resolve("vendor").resolve("bundle").resolve("jruby").resolve("2.6.0").
                     toFile().getAbsolutePath();
             final RubyHash environment = RubyUtil.RUBY.getENV();
             environment.put("GEM_HOME", gems);

qa/integration/rspec.rb

@@ -20,7 +20,7 @@
 ::Gem.clear_paths
 
 ENV['GEM_HOME'] = ENV['GEM_PATH'] = ::File.expand_path(
-    ::File.join(__FILE__, "..", "..", "..", "build", "qa", "integration", "vendor", "jruby", "2.5.0")
+    ::File.join(__FILE__, "..", "..", "..", "build", "qa", "integration", "vendor", "jruby", "2.6.0")
 )
 
 ::Gem.paths = ENV

qa/integration/specs/command_line_spec.rb

@@ -40,7 +40,7 @@
     execute = @logstash.run
     lines = execute.stderr_and_stdout.split("\n")
     expect(lines.shift).to match(/^(Using system java)|(Using bundled JDK)|(Using LS_JAVA_HOME defined java):/)
-    while (up_line = lines.shift).match(/OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated|warning: ignoring JAVA_TOOL_OPTIONS/) do end
+    while (up_line = lines.shift).match(/OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated|warning: ignoring JAVA_TOOL_OPTIONS|warning: already initialized constant Socket::Constants/) do end
     expect(up_line).to match(/^Sending Logstash logs to/)
   end
 end

rakelib/artifacts.rake

@@ -346,6 +346,8 @@ namespace "artifact" do
   end
 
   task "generate_build_metadata" do
+    require 'time'
+
     return if defined?(BUILD_METADATA_FILE)
     BUILD_METADATA_FILE = Tempfile.new('build.rb')
     BUILD_DATE=Time.now.iso8601

rakelib/compile.rake

@@ -16,6 +16,7 @@
 # under the License.
 
 rule ".rb" => ".treetop" do |task, args|
+  require 'logstash/patches/polyglot'
   require "treetop"
   compiler = Treetop::Compiler::GrammarCompiler.new
   compiler.compile(task.source, task.name)

rubyUtils.gradle

@@ -25,7 +25,7 @@ buildscript {
     dependencies {
         classpath 'org.yaml:snakeyaml:1.29'
         classpath "de.undercouch:gradle-download-task:4.0.4"
-        classpath "org.jruby:jruby-complete:9.2.20.1"
+        classpath "org.jruby:jruby-complete:9.3.4.0"
     }
 }
 
@@ -139,7 +139,7 @@ void buildGem(File projectDir, File buildDir, String gemspec) {
 void rake(File projectDir, File buildDir, String task) {
     executeJruby projectDir, buildDir, { ScriptingContainer jruby ->
         jruby.currentDirectory = projectDir
-        jruby.runScriptlet("require 'rake'")
+        jruby.runScriptlet("require 'rake'; require 'time'")
         jruby.runScriptlet("""
                 rake = Rake.application
                 rake.init
@@ -168,7 +168,7 @@ void setupJruby(File projectDir, File buildDir) {
 Object executeJruby(File projectDir, File buildDir, Closure<?> /* Object*/ block) {
     def jruby = new ScriptingContainer()
     def env = jruby.environment
-    def gemDir = "${projectDir}/vendor/bundle/jruby/2.5.0".toString()
+    def gemDir = "${projectDir}/vendor/bundle/jruby/2.6.0".toString()
     env.put "USE_RUBY", "1"
     env.put "GEM_HOME", gemDir
     env.put "GEM_SPEC_CACHE", "${buildDir}/cache".toString()