Plugins: Add plugin extension capabilities (elastic#27881)

This commit adds the infrastructure to plugin building and loading to
allow one plugin to extend another. That is, one plugin may extend
another by the "parent" plugin allowing itself to be extended through
java SPI. When all plugins extending a plugin are finished loading, the
"parent" plugin has a callback (through the ExtensiblePlugin interface)
allowing it to reload SPI.

This commit also adds an example plugin which uses as-yet implemented
extensibility (adding to the painless whitelist).

build.gradle

@@ -139,7 +139,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 allprojects {
-  ext.bwc_tests_enabled = true
+  // TODO: re-enable after https://github.com/elastic/elasticsearch/pull/27881 is backported
+  ext.bwc_tests_enabled = false
 }
 
 task verifyBwcTestsEnabled {

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy

@@ -39,6 +39,10 @@ class PluginPropertiesExtension {
     @Input
     String classname
 
+    /** Other plugins this plugin extends through SPI */
+    @Input
+    List<String> extendedPlugins = []
+
     @Input
     boolean hasNativeController = false
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy

@@ -80,6 +80,7 @@ class PluginPropertiesTask extends Copy {
             'elasticsearchVersion': stringSnap(VersionProperties.elasticsearch),
             'javaVersion': project.targetCompatibility as String,
             'classname': extension.classname,
+            'extendedPlugins': extension.extendedPlugins.join(','),
             'hasNativeController': extension.hasNativeController,
             'requiresKeystore': extension.requiresKeystore
         ]

buildSrc/src/main/resources/plugin-descriptor.properties

@@ -40,6 +40,9 @@ java.version=${javaVersion}
 elasticsearch.version=${elasticsearchVersion}
 ### optional elements for plugins:
 #
+#  'extended.plugins': other plugins this plugin extends through SPI
+extended.plugins=${extendedPlugins}
+#
 # 'has.native.controller': whether or not the plugin has a native controller
 has.native.controller=${hasNativeController}
 #

core/build.gradle

@@ -38,6 +38,9 @@ archivesBaseName = 'elasticsearch'
 
 dependencies {
 
+  compileOnly project(':libs:plugin-classloader')
+  testRuntime project(':libs:plugin-classloader')
+
   // lucene
   compile "org.apache.lucene:lucene-core:${versions.lucene}"
   compile "org.apache.lucene:lucene-analyzers-common:${versions.lucene}"

core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java

@@ -33,6 +33,7 @@
 import java.security.ProtectionDomain;
 import java.util.Collections;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Predicate;
 
 /** custom policy for union of static and dynamic permissions */
@@ -49,9 +50,9 @@ final class ESPolicy extends Policy {
     final PermissionCollection dynamic;
     final Map<String,Policy> plugins;
 
-    ESPolicy(PermissionCollection dynamic, Map<String,Policy> plugins, boolean filterBadDefaults) {
-        this.template = Security.readPolicy(getClass().getResource(POLICY_RESOURCE), JarHell.parseClassPath());
-        this.untrusted = Security.readPolicy(getClass().getResource(UNTRUSTED_RESOURCE), Collections.emptySet());
+    ESPolicy(Map<String, URL> codebases, PermissionCollection dynamic, Map<String,Policy> plugins, boolean filterBadDefaults) {
+        this.template = Security.readPolicy(getClass().getResource(POLICY_RESOURCE), codebases);
+        this.untrusted = Security.readPolicy(getClass().getResource(UNTRUSTED_RESOURCE), Collections.emptyMap());
         if (filterBadDefaults) {
             this.system = new SystemPolicy(Policy.getPolicy());
         } else {

core/src/main/java/org/elasticsearch/bootstrap/Security.java

@@ -48,10 +48,13 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.bootstrap.FilePermissionUtils.addDirectoryPath;
 import static org.elasticsearch.bootstrap.FilePermissionUtils.addSingleFilePath;
@@ -116,7 +119,8 @@ private Security() {}
     static void configure(Environment environment, boolean filterBadDefaults) throws IOException, NoSuchAlgorithmException {
 
         // enable security policy: union of template and environment-based paths, and possibly plugin permissions
-        Policy.setPolicy(new ESPolicy(createPermissions(environment), getPluginPermissions(environment), filterBadDefaults));
+        Map<String, URL> codebases = getCodebaseJarMap(JarHell.parseClassPath());
+        Policy.setPolicy(new ESPolicy(codebases, createPermissions(environment), getPluginPermissions(environment), filterBadDefaults));
 
         // enable security manager
         final String[] classesThatCanExit =
@@ -130,6 +134,27 @@ static void configure(Environment environment, boolean filterBadDefaults) throws
         selfTest();
     }
 
+    /**
+     * Return a map from codebase name to codebase url of jar codebases used by ES core.
+     */
+    @SuppressForbidden(reason = "find URL path")
+    static Map<String, URL> getCodebaseJarMap(Set<URL> urls) {
+        Map<String, URL> codebases = new LinkedHashMap<>(); // maintain order
+        for (URL url : urls) {
+            try {
+                String fileName = PathUtils.get(url.toURI()).getFileName().toString();
+                if (fileName.endsWith(".jar") == false) {
+                    // tests :(
+                    continue;
+                }
+                codebases.put(fileName, url);
+            } catch (URISyntaxException e) {
+                throw new RuntimeException(e);
+            }
+        }
+        return codebases;
+    }
+
     /**
      * Sets properties (codebase URLs) for policy files.
      * we look for matching plugins and set URLs to fit
@@ -174,7 +199,7 @@ static Map<String,Policy> getPluginPermissions(Environment environment) throws I
                 }
 
                 // parse the plugin's policy file into a set of permissions
-                Policy policy = readPolicy(policyFile.toUri().toURL(), codebases);
+                Policy policy = readPolicy(policyFile.toUri().toURL(), getCodebaseJarMap(codebases));
 
                 // consult this policy for each of the plugin's jars:
                 for (URL url : codebases) {
@@ -197,21 +222,20 @@ static Map<String,Policy> getPluginPermissions(Environment environment) throws I
      * would map to full URL.
      */
     @SuppressForbidden(reason = "accesses fully qualified URLs to configure security")
-    static Policy readPolicy(URL policyFile, Set<URL> codebases) {
+    static Policy readPolicy(URL policyFile, Map<String, URL> codebases) {
         try {
             List<String> propertiesSet = new ArrayList<>();
             try {
                 // set codebase properties
-                for (URL url : codebases) {
-                    String fileName = PathUtils.get(url.toURI()).getFileName().toString();
-                    if (fileName.endsWith(".jar") == false) {
-                        continue; // tests :(
-                    }
+                for (Map.Entry<String,URL> codebase : codebases.entrySet()) {
+                    String name = codebase.getKey();
+                    URL url = codebase.getValue();
+
                     // We attempt to use a versionless identifier for each codebase. This assumes a specific version
                     // format in the jar filename. While we cannot ensure all jars in all plugins use this format, nonconformity
                     // only means policy grants would need to include the entire jar filename as they always have before.
-                    String property = "codebase." + fileName;
-                    String aliasProperty = "codebase." + fileName.replaceFirst("-\\d+\\.\\d+.*\\.jar", "");
+                    String property = "codebase." + name;
+                    String aliasProperty = "codebase." + name.replaceFirst("-\\d+\\.\\d+.*\\.jar", "");
                     if (aliasProperty.equals(property) == false) {
                         propertiesSet.add(aliasProperty);
                         String previous = System.setProperty(aliasProperty, url.toString());

core/src/main/java/org/elasticsearch/plugins/ExtensiblePlugin.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+/**
+ * An extension point for {@link Plugin} implementations to be themselves extensible.
+ *
+ * This class provides a callback for extensible plugins to be informed of other plugins
+ * which extend them.
+ */
+public interface ExtensiblePlugin {
+
+    /**
+     * Reload any SPI implementations from the given classloader.
+     */
+    default void reloadSPI(ClassLoader loader) {}
+}

core/src/main/java/org/elasticsearch/plugins/PluginInfo.java

@@ -22,17 +22,20 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.bootstrap.JarHell;
 import org.elasticsearch.common.Booleans;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ToXContent.Params;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Properties;
@@ -51,6 +54,7 @@ public class PluginInfo implements Writeable, ToXContentObject {
     private final String description;
     private final String version;
     private final String classname;
+    private final List<String> extendedPlugins;
     private final boolean hasNativeController;
     private final boolean requiresKeystore;
 
@@ -61,15 +65,17 @@ public class PluginInfo implements Writeable, ToXContentObject {
      * @param description         a description of the plugin
      * @param version             the version of Elasticsearch the plugin is built for
      * @param classname           the entry point to the plugin
+     * @param extendedPlugins     other plugins this plugin extends through SPI
      * @param hasNativeController whether or not the plugin has a native controller
      * @param requiresKeystore    whether or not the plugin requires the elasticsearch keystore to be created
      */
     public PluginInfo(String name, String description, String version, String classname,
-                      boolean hasNativeController, boolean requiresKeystore) {
+                      List<String> extendedPlugins, boolean hasNativeController, boolean requiresKeystore) {
         this.name = name;
         this.description = description;
         this.version = version;
         this.classname = classname;
+        this.extendedPlugins = Collections.unmodifiableList(extendedPlugins);
         this.hasNativeController = hasNativeController;
         this.requiresKeystore = requiresKeystore;
     }
@@ -85,6 +91,11 @@ public PluginInfo(final StreamInput in) throws IOException {
         this.description = in.readString();
         this.version = in.readString();
         this.classname = in.readString();
+        if (in.getVersion().onOrAfter(Version.V_6_2_0)) {
+            extendedPlugins = in.readList(StreamInput::readString);
+        } else {
+            extendedPlugins = Collections.emptyList();
+        }
         if (in.getVersion().onOrAfter(Version.V_5_4_0)) {
             hasNativeController = in.readBoolean();
         } else {
@@ -103,6 +114,9 @@ public void writeTo(final StreamOutput out) throws IOException {
         out.writeString(description);
         out.writeString(version);
         out.writeString(classname);
+        if (out.getVersion().onOrAfter(Version.V_6_2_0)) {
+            out.writeStringList(extendedPlugins);
+        }
         if (out.getVersion().onOrAfter(Version.V_5_4_0)) {
             out.writeBoolean(hasNativeController);
         }
@@ -176,6 +190,14 @@ public static PluginInfo readFromProperties(final Path path) throws IOException
                     "property [classname] is missing for plugin [" + name + "]");
         }
 
+        final String extendedString = propsMap.remove("extended.plugins");
+        final List<String> extendedPlugins;
+        if (extendedString == null) {
+            extendedPlugins = Collections.emptyList();
+        } else {
+            extendedPlugins = Arrays.asList(Strings.delimitedListToStringArray(extendedString, ","));
+        }
+
         final String hasNativeControllerValue = propsMap.remove("has.native.controller");
         final boolean hasNativeController;
         if (hasNativeControllerValue == null) {
@@ -216,7 +238,7 @@ public static PluginInfo readFromProperties(final Path path) throws IOException
             throw new IllegalArgumentException("Unknown properties in plugin descriptor: " + propsMap.keySet());
         }
 
-        return new PluginInfo(name, description, version, classname, hasNativeController, requiresKeystore);
+        return new PluginInfo(name, description, version, classname, extendedPlugins, hasNativeController, requiresKeystore);
     }
 
     /**
@@ -246,6 +268,15 @@ public String getClassname() {
         return classname;
     }
 
+    /**
+     * Other plugins this plugin extends through SPI.
+     *
+     * @return the names of the plugins extended
+     */
+    public List<String> getExtendedPlugins() {
+        return extendedPlugins;
+    }
+
     /**
      * The version of Elasticsearch the plugin was built for.
      *
@@ -281,6 +312,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field("version", version);
             builder.field("description", description);
             builder.field("classname", classname);
+            builder.field("extended_plugins", extendedPlugins);
             builder.field("has_native_controller", hasNativeController);
             builder.field("requires_keystore", requiresKeystore);
         }
@@ -316,6 +348,7 @@ public String toString() {
                 .append("Version: ").append(version).append("\n")
                 .append("Native Controller: ").append(hasNativeController).append("\n")
                 .append("Requires Keystore: ").append(requiresKeystore).append("\n")
+                .append("Extended Plugins: ").append(extendedPlugins).append("\n")
                 .append(" * Classname: ").append(classname);
         return information.toString();
     }

core/src/main/java/org/elasticsearch/plugins/DummyPluginInfo.java → core/src/main/java/org/elasticsearch/plugins/PluginLoaderIndirection.java

@@ -16,18 +16,18 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.plugins;
 
-public class DummyPluginInfo extends PluginInfo {
+import java.util.List;
 
-    private DummyPluginInfo(String name, String description, String version, String classname) {
-        super(name, description, version, classname, false, false);
-    }
+/**
+ * This class exists solely as an intermediate layer to avoid causing PluginsService
+ * to load ExtendedPluginsClassLoader when used in the transport client.
+ */
+class PluginLoaderIndirection {
 
-    public static final DummyPluginInfo INSTANCE =
-            new DummyPluginInfo(
-                    "dummy_plugin_name",
-                    "dummy plugin description",
-                    "dummy_plugin_version",
-                    "DummyPluginName");
+    static ClassLoader createLoader(ClassLoader parent, List<ClassLoader> extendedLoaders) {
+        return ExtendedPluginsClassLoader.create(parent, extendedLoaders);
+    }
 }