Refactor: drop redundant (jruby-complete.jar) dependency (elastic#13159)

Co-authored-by: João Duarte <[email protected]>

bin/logstash

@@ -56,6 +56,5 @@ if [ "$1" = "-V" ] || [ "$1" = "--version" ]; then
   fi
   echo "logstash $LOGSTASH_VERSION"
 else
-  CLASSPATH="$(setup_classpath $LOGSTASH_JARS)"
   exec "${JAVACMD}" ${JAVA_OPTS} -cp "${CLASSPATH}" org.logstash.Logstash "$@"
 fi

bin/logstash.bat

@@ -33,6 +33,10 @@ for /F "usebackq tokens=1-2* delims= " %%A in (!params!) do (
 	)
 )
 
+rem setup CLASSPATH for Java process
+set "JRUBY_HOME=%LS_HOME%\vendor\jruby"
+
+set "CLASSPATH=%JRUBY_HOME%\lib\jruby.jar"
 for %%i in ("%LS_HOME%\logstash-core\lib\jars\*.jar") do (
 	call :concat "%%i"
 )

bin/logstash.lib.sh

@@ -47,6 +47,7 @@ export LS_HOME="${LOGSTASH_HOME}"
 SINCEDB_DIR="${LOGSTASH_HOME}"
 export SINCEDB_DIR
 LOGSTASH_JARS=${LOGSTASH_HOME}/logstash-core/lib/jars
+JRUBY_HOME="${LOGSTASH_HOME}/vendor/jruby"
 
 # iterate over the command line args and look for the argument
 # after --path.settings to see if the jvm.options file is in
@@ -77,14 +78,18 @@ setup_bundled_jdk_part() {
 
 # Accepts 1 parameter which is the path the directory where logstash jar are contained.
 setup_classpath() {
-  local jar_directory="${1?jar directory required}"
-  local classpath
+  local classpath="${1?classpath (and jar_directory) required}"
+  local jar_directory="${2}"
   for J in $(cd "${jar_directory}"; ls *.jar); do
     classpath=${classpath}${classpath:+:}${jar_directory}/${J}
   done
   echo "${classpath}"
 }
 
+# set up CLASSPATH once (we start more than one java process)
+CLASSPATH="${JRUBY_HOME}/lib/jruby.jar"
+CLASSPATH="$(setup_classpath $CLASSPATH $LOGSTASH_JARS)"
+
 setup_java() {
   # set the path to java into JAVACMD which will be picked up by JRuby to launch itself
   if [ -z "$JAVACMD" ]; then
@@ -137,20 +142,17 @@ setup_java() {
   # jruby launcher uses JAVACMD as its java executable and JAVA_OPTS as the JVM options
   export JAVACMD
 
-  CLASSPATH="$(setup_classpath $LOGSTASH_JARS)"
-
   JAVA_OPTS=`exec "${JAVACMD}" -cp "${CLASSPATH}" org.logstash.launchers.JvmOptionsParser "$LOGSTASH_HOME" "$LS_JVM_OPTS"`
   EXIT_CODE=$?
   if [ $EXIT_CODE -ne 0 ]; then
     exit $EXIT_CODE
   fi
 
-  unset CLASSPATH
   export JAVA_OPTS
 }
 
 setup_vendored_jruby() {
-  JRUBY_BIN="${LOGSTASH_HOME}/vendor/jruby/bin/jruby"
+  JRUBY_BIN="${JRUBY_HOME}/bin/jruby"
 
   if [ ! -f "${JRUBY_BIN}" ] ; then
     echo "Unable to find JRuby."

build.gradle

@@ -872,9 +872,11 @@ if (System.getenv('OSS') != 'true') {
   }
 }
 
- tasks.register("runXPackUnitTests"){
-   dependsOn ":logstash-xpack:rubyTests"
+ tasks.register("runXPackUnitTests") {
+     dependsOn copyPluginTestAlias
+     dependsOn ":logstash-xpack:rubyTests"
  }
  tasks.register("runXPackIntegrationTests"){
-   dependsOn ":logstash-xpack:rubyIntegrationTests"
+     dependsOn copyPluginTestAlias
+     dependsOn ":logstash-xpack:rubyIntegrationTests"
  }

buildSrc/src/main/groovy/org/logstash/gradle/tooling/ListProjectDependencies.groovy

@@ -34,6 +34,13 @@ class ListProjectDependencies extends DefaultTask {
         moduleDeps.addAll(project.configurations.runtimeOnly.getAllDependencies())
 
         def depsInGAV = moduleDeps.collect {dep ->
+
+            // SelfResolvingDependency has no group or version information
+            if (!dep.group && !dep.version && dep.reason) {
+                def m = dep.reason =~ /DEPENDENCY: ([\w\.\-]+:[\w\.\-]+:[\w\.\-]+)/
+                if (m.matches()) return m[0][1]
+            }
+
             "${dep.group}:${dep.name}:${dep.version}"
         }
 

ci/integration_tests.sh

@@ -9,7 +9,6 @@ export GRADLE_OPTS="-Xmx4g -Dorg.gradle.jvmargs=-Xmx4g -Dorg.gradle.daemon=false
 
 export SPEC_OPTS="--order rand --format documentation"
 export CI=true
-export OSS=true
 
 if [ -n "$BUILD_JAVA_HOME" ]; then
   GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME"

ci/unit_tests.sh

@@ -9,7 +9,6 @@ export GRADLE_OPTS="-Xmx4g -Dorg.gradle.jvmargs=-Xmx4g -Dorg.gradle.daemon=false
 
 export SPEC_OPTS="--order rand --format documentation"
 export CI=true
-export OSS=true
 export TEST_DEBUG=true
 
 if [ -n "$BUILD_JAVA_HOME" ]; then

lib/bootstrap/rspec.rb

@@ -20,13 +20,18 @@
 require "logstash-core"
 require "logstash/environment"
 
-$LOAD_PATH.unshift(File.join(LogStash::Environment::LOGSTASH_CORE, "spec"))
+# Bundler + gemspec already setup $LOAD_PATH << '.../lib'
+# but since we load specs from 2 locations we need to hook up these:
+[ LogStash::Environment::LOGSTASH_HOME, LogStash::Environment::LOGSTASH_CORE ].each do |path|
+  spec_path = File.join(path, "spec")
+  $LOAD_PATH.unshift(spec_path) unless $LOAD_PATH.include?(spec_path)
+end
 
 require "rspec/core"
 require "rspec"
 require 'ci/reporter/rake/rspec_loader'
 
-RSpec.world.reset # if multiple rspec runs occur in a single process, the RSpec "world" state needs to be reset.
+RSpec.clear_examples # if multiple rspec runs occur in a single process, the RSpec "world" state needs to be reset.
 
 status = RSpec::Core::Runner.run(ARGV.empty? ? ($JUNIT_ARGV || ["spec"]) : ARGV).to_i
 if ENV["IS_JUNIT_RUN"]

logstash-core/build.gradle

@@ -17,18 +17,16 @@
  * under the License.
  */
 
-import java.nio.file.Files
-import java.nio.file.Paths
 import org.yaml.snakeyaml.Yaml
 
 // fetch version from Logstash's main versions.yml file
 def versionMap = (Map) (new Yaml()).load(new File("$projectDir/../versions.yml").text)
 
 description = """Logstash Core Java"""
-version = versionMap['logstash-core']
-String jrubyVersion = versionMap['jruby']['version']
+
 String jacksonVersion = versionMap['jackson']
 String jacksonDatabindVersion = versionMap['jackson-databind']
+String jrubyVersion = versionMap['jruby']['version']
 
 String log4jVersion = '2.17.1'
 
@@ -61,7 +59,9 @@ tasks.register("javadocJar", Jar) {
 
 tasks.register("copyRuntimeLibs", Copy) {
     into project.file('lib/jars/')
-    from configurations.compileClasspath, configurations.runtimeClasspath
+    from configurations.runtimeClasspath {
+        exclude { it.file.name == 'jruby.jar' }
+    }
 }
 
 // copy jar file into the gem lib dir but without the version number in filename
@@ -90,6 +90,7 @@ configurations.archives {
 }
 
 tasks.register("javaTests", Test) {
+    dependsOn ':bootstrap'
     exclude '/org/logstash/RSpecTests.class'
     exclude '/org/logstash/config/ir/ConfigCompilerTest.class'
     exclude '/org/logstash/config/ir/CompiledPipelineTest.class'
@@ -105,7 +106,7 @@ tasks.register("javaTests", Test) {
 tasks.register("rubyTests", Test) {
     inputs.files fileTree("${projectDir}/lib")
     inputs.files fileTree("${projectDir}/spec")
-    systemProperty 'logstash.core.root.dir', projectDir.absolutePath
+    systemProperty 'logstash.root.dir', projectDir.parent
     include '/org/logstash/RSpecTests.class'
     include '/org/logstash/config/ir/ConfigCompilerTest.class'
     include '/org/logstash/config/ir/CompiledPipelineTest.class'
@@ -151,10 +152,10 @@ idea {
     }
 }
 
-def customJRubyDir = project.hasProperty("custom.jruby.path") ? project.property("custom.jruby.path") : ""
-def customJRubyVersion = customJRubyDir == "" ? "" : Files.readAllLines(Paths.get(customJRubyDir, "VERSION")).get(0).trim()
-
 dependencies {
+    api(files("../vendor/jruby/lib/jruby.jar") { // jruby-core.jar
+        builtBy ':downloadAndInstallJRuby'
+    }) { because "DEPENDENCY: org.jruby:jruby-core:${jrubyVersion}" } // meta-data for generateLicenseReport
     implementation "org.apache.logging.log4j:log4j-api:${log4jVersion}"
     annotationProcessor "org.apache.logging.log4j:log4j-core:${log4jVersion}"
     api "org.apache.logging.log4j:log4j-core:${log4jVersion}"
@@ -177,11 +178,6 @@ dependencies {
     implementation 'org.codehaus.janino:janino:3.1.0'
     implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jacksonVersion}"
     implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jacksonVersion}"
-    if (customJRubyDir == "") {
-        api "org.jruby:jruby-complete:${jrubyVersion}"
-    } else {
-        api files(customJRubyDir + "/maven/jruby-complete/target/jruby-complete-${customJRubyVersion}.jar")
-    }
     implementation group: 'com.google.guava', name: 'guava', version: '24.1.1-jre'
     // WARNING: DO NOT UPGRADE "google-java-format"
     // later versions require GPL licensed code in javac-shaded that is

logstash-core/lib/logstash/environment.rb

@@ -145,7 +145,7 @@ module Environment
   module Environment
     extend self
 
-    LOGSTASH_CORE = ::File.expand_path(::File.join(::File.dirname(__FILE__), "..", ".."))
+    LOGSTASH_CORE = ::File.expand_path(::File.join("..", ".."), ::File.dirname(__FILE__))
     LOGSTASH_ENV = (ENV["LS_ENV"] || 'production').to_s.freeze
 
     LINUX_OS_RE = /linux/

logstash-core/src/main/java/org/logstash/Logstash.java

@@ -38,6 +38,8 @@
 import org.jruby.exceptions.RaiseException;
 import org.jruby.runtime.builtin.IRubyObject;
 
+import javax.annotation.Nullable;
+
 /**
  * Logstash Main Entrypoint.
  */
@@ -197,6 +199,40 @@ public void close() {
         ruby.tearDown(false);
     }
 
+    /**
+     * Initialize a runtime configuration.
+     * @param lsHome the LOGSTASH_HOME
+     * @param args extra arguments (ARGV) to process
+     * @return a runtime configuration instance
+     */
+    public static RubyInstanceConfig initRubyConfig(final Path lsHome,
+                                                    final String... args) {
+        return initRubyConfigImpl(lsHome, safePath(lsHome, "vendor", "jruby"), args);
+    }
+
+    /**
+     * Initialize a runtime configuration.
+     * @param lsHome the LOGSTASH_HOME
+     * @param args extra arguments (ARGV) to process
+     * @return a runtime configuration instance
+     */
+    public static RubyInstanceConfig initRubyConfig(final Path lsHome,
+                                                    final Path currentDir,
+                                                    final String... args) {
+
+        return initRubyConfigImpl(currentDir, safePath(lsHome, "vendor", "jruby"), args);
+    }
+
+    private static RubyInstanceConfig initRubyConfigImpl(@Nullable final Path currentDir,
+                                                     final String jrubyHome,
+                                                     final String[] args) {
+        final RubyInstanceConfig config = new RubyInstanceConfig();
+        if (currentDir != null) config.setCurrentDirectory(currentDir.toString());
+        config.setJRubyHome(jrubyHome);
+        config.processArguments(args);
+        return config;
+    }
+
     /**
      * Sets up the correct {@link RubyInstanceConfig} for a given Logstash installation and set of
      * CLI arguments.
@@ -209,9 +245,7 @@ private static RubyInstanceConfig buildConfig(final Path home, final String[] ar
         System.arraycopy(args, 0, arguments, 2, args.length);
         arguments[0] = safePath(home, "lib", "bootstrap", "environment.rb");
         arguments[1] = safePath(home, "logstash-core", "lib", "logstash", "runner.rb");
-        final RubyInstanceConfig config = new RubyInstanceConfig();
-        config.processArguments(arguments);
-        return config;
+        return initRubyConfig(home, arguments);
     }
 
     /**

logstash-core/src/main/java/org/logstash/secret/store/backend/JavaKeyStore.java

@@ -23,7 +23,11 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.logstash.secret.SecretIdentifier;
-import org.logstash.secret.store.*;
+import org.logstash.secret.store.SecretStore;
+import org.logstash.secret.store.SecretStoreFactory;
+import org.logstash.secret.store.SecretStoreException;
+import org.logstash.secret.store.SecretStoreUtil;
+import org.logstash.secret.store.SecureConfig;
 
 import javax.crypto.SecretKey;
 import javax.crypto.SecretKeyFactory;

logstash-core/src/test/java/org/logstash/AccessorsTest.java

@@ -32,7 +32,7 @@
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
-public class AccessorsTest {
+public class AccessorsTest extends RubyTestBase {
 
     @Test
     public void testBareGet() throws Exception {

logstash-core/src/test/java/org/logstash/ClonerTest.java

@@ -28,7 +28,7 @@
 
 import static org.junit.Assert.*;
 
-public class ClonerTest {
+public class ClonerTest extends RubyTestBase {
     @Test
     public void testRubyStringCloning() {
         String javaString = "fooBar";

logstash-core/src/test/java/org/logstash/DLQEntryTest.java

@@ -48,7 +48,7 @@
 import static org.hamcrest.core.IsEqual.equalTo;
 import static org.junit.Assert.assertNotNull;
 
-public class DLQEntryTest {
+public class DLQEntryTest extends RubyTestBase {
     @Test
     public void testConstruct() throws Exception {
         Event event = new Event(Collections.singletonMap("key", "value"));

logstash-core/src/test/java/org/logstash/EventTest.java

@@ -44,7 +44,7 @@
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
-public final class EventTest {
+public final class EventTest extends RubyTestBase {
 
     @Test
     public void queueableInterfaceRoundTrip() throws Exception {

logstash-core/src/test/java/org/logstash/FieldReferenceTest.java

@@ -34,7 +34,7 @@
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
-public final class FieldReferenceTest {
+public final class FieldReferenceTest extends RubyTestBase {
     @Before
     @After
     public void clearInternalCaches() {

logstash-core/src/test/java/org/logstash/JavafierTest.java

@@ -26,7 +26,7 @@
 import java.math.BigInteger;
 import static org.junit.Assert.assertEquals;
 
-public class JavafierTest {
+public class JavafierTest extends RubyTestBase {
 
     @Test
     public void testRubyBignum() {

logstash-core/src/test/java/org/logstash/RSpecTests.java

@@ -22,11 +22,9 @@
 
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.Collection;
 
-import org.assertj.core.util.Files;
 import org.jruby.runtime.builtin.IRubyObject;
 import org.junit.Assert;
 import org.junit.Test;
@@ -39,13 +37,13 @@
  * Runs the Logstash RSpec suit.
  */
 @RunWith(Parameterized.class)
-public final class RSpecTests {
+public class RSpecTests extends RubyTestBase {
 
     @Parameters(name="{0}")
     public static Collection<Object[]> data() {
         return Arrays.asList(new Object[][] {
-                { "compliance", "spec/compliance/**/*_spec.rb"},
-                { "core tests", "spec/unit/**/*_spec.rb,logstash-core/spec/**/*_spec.rb" }
+                { "compliance", "spec/compliance/**/*_spec.rb" },
+                { "core tests", "spec/unit/**/*_spec.rb" + ',' + "logstash-core/spec/**/*_spec.rb" }
         });
     }
 
@@ -57,17 +55,15 @@ public static Collection<Object[]> data() {
 
     @Test
     public void rspecTests() throws Exception {
-        final String root = Files.currentFolder().getParent();
         RubyUtil.RUBY.getENV().put("IS_JUNIT_RUN", "true");
-        RubyUtil.RUBY.setCurrentDirectory(root);
         RubyUtil.RUBY.getGlobalVariables().set(
             "$JUNIT_ARGV",
             Rubyfier.deep(
                 RubyUtil.RUBY, Arrays.asList(
                     "-fd", "--pattern", specGlob
                 ))
         );
-        final Path rspec = Paths.get(root, "lib/bootstrap/rspec.rb");
+        final Path rspec = LS_HOME.resolve("lib").resolve("bootstrap").resolve("rspec.rb");
         final IRubyObject result = RubyUtil.RUBY.executeScript(
             new String(java.nio.file.Files.readAllBytes(rspec), StandardCharsets.UTF_8),
             rspec.toFile().getAbsolutePath()