Skip to content

Commit

Permalink
GEODE-2290: Limit scanning of deployed jars
Browse files Browse the repository at this point in the history 
 - Uses fast-classpath-scanner to scan jars for classes containing Functions without eagerly loading all classes in the jar.
  • Loading branch information
@jaredjstewart
jaredjstewart committed Apr 16, 2017
1 parent 8f9624d commit 6f7f943
Show file tree
Hide file tree
Showing 20 changed files with 316 additions and 375 deletions.
1 change: 1 addition & 0 deletions geode-assembly/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ def cp = {
it.contains('commons-io') ||
it.contains('commons-lang') ||
it.contains('commons-logging') ||
it.contains('fast-classpath-scanner') ||
it.contains('fastutil') ||
it.contains('jackson-annotations') ||
it.contains('jackson-core') ||
Expand Down
1 change: 1 addition & 0 deletions geode-assembly/src/test/resources/expected_jars.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ commons-io
commons-lang
commons-logging
commons-modeler
fast-classpath-scanner
fastutil
findbugs-annotations
gfsh-dependencies.jar
Expand Down
15 changes: 10 additions & 5 deletions geode-core/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ configurations {

dependencies {
// Source Dependencies
// External
// External
provided files("${System.getProperty('java.home')}/../lib/tools.jar")
compile 'com.github.stephenc.findbugs:findbugs-annotations:' + project.'stephenc-findbugs.version'
compile 'org.jgroups:jgroups:' + project.'jgroups.version'
Expand Down Expand Up @@ -104,17 +104,22 @@ dependencies {
}
compile ('org.iq80.snappy:snappy:' + project.'snappy-java.version') {
ext.optional = true
}
}

compile 'org.apache.shiro:shiro-core:' + project.'shiro.version'
// This is only added since shiro is using an old version of beanutils and we want
// to use a standard version. Once shiro deps are updated, remove this explicit dependency
// in favor of a transitive dependency on beanutils.
compile 'commons-beanutils:commons-beanutils:' + project.'commons-beanutils.version'


// https://mvnrepository.com/artifact/io.github.lukehutch/fast-classpath-scanner
compile 'io.github.lukehutch:fast-classpath-scanner:' + project.'fast-classpath-scanner.version'



compile project(':geode-common')
compile project(':geode-json')

jcaCompile sourceSets.main.output

testCompile project(':geode-junit')
Expand Down Expand Up @@ -181,7 +186,7 @@ jar {

from sourceSets.main.output
from sourceSets.jca.output

exclude 'org/apache/geode/management/internal/web/**'
exclude 'org/apache/geode/internal/i18n/StringIdResourceBundle_ja.txt'
exclude 'org/apache/geode/admin/doc-files/ds4_0.dtd'
Expand Down
97 changes: 45 additions & 52 deletions geode-core/src/main/java/org/apache/geode/internal/DeployedJar.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@
*/
package org.apache.geode.internal;

import io.github.lukehutch.fastclasspathscanner.FastClasspathScanner;
import io.github.lukehutch.fastclasspathscanner.scanner.ScanResult;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
Expand All @@ -25,7 +28,7 @@
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Modifier;
import java.net.URL;
import java.nio.channels.FileLock;
import java.net.URLClassLoader;
import java.nio.file.Files;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
Expand All @@ -37,7 +40,6 @@
import java.util.jar.JarEntry;
import java.util.jar.JarInputStream;

import org.apache.geode.cache.Cache;
import org.apache.logging.log4j.Logger;

import org.apache.geode.cache.CacheClosedException;
Expand All @@ -48,7 +50,6 @@
import org.apache.geode.internal.cache.GemFireCacheImpl;
import org.apache.geode.internal.logging.LogService;
import org.apache.geode.pdx.internal.TypeRegistry;
import sun.nio.ch.ChannelInputStream;

/**
* ClassLoader for a single JAR file.
Expand Down Expand Up @@ -103,7 +104,7 @@ public DeployedJar(File versionedJarFile, final String jarName, byte[] jarBytes)
+ ", was modified prior to obtaining a lock: " + jarName);
}

if (!isValidJarContent(getJarContent())) {
if (!hasValidJarContent(getJarContent())) {
throw new IllegalArgumentException(
"File does not contain valid JAR content: " + versionedJarFile.getAbsolutePath());
}
Expand Down Expand Up @@ -147,23 +148,10 @@ private static boolean hasValidJarContent(final InputStream inputStream) {
* @param jarBytes Bytes of data to be validated.
* @return True if the data has JAR content, false otherwise
*/
public static boolean isValidJarContent(final byte[] jarBytes) {
public static boolean hasValidJarContent(final byte[] jarBytes) {
return hasValidJarContent(new ByteArrayInputStream(jarBytes));
}

/**
* Peek into the JAR data and make sure that it is valid JAR content.
*
* @param jarFile File whose contents should be validated.
* @return True if the data has JAR content, false otherwise
*/
public static boolean hasValidJarContent(final File jarFile) {
try {
return hasValidJarContent(new FileInputStream(jarFile));
} catch (IOException ioex) {
return false;
}
}

/**
* Scan the JAR file and attempt to load all classes and register any function classes found.
Expand All @@ -184,37 +172,42 @@ public synchronized void loadClassesAndRegisterFunctions() throws ClassNotFoundE

JarInputStream jarInputStream = null;
try {
List<String> functionClasses = findFunctionsInThisJar();

jarInputStream = new JarInputStream(byteArrayInputStream);
JarEntry jarEntry = jarInputStream.getNextJarEntry();

while (jarEntry != null) {
if (jarEntry.getName().endsWith(".class")) {
if (isDebugEnabled) {
logger.debug("Attempting to load class: {}, from JAR file: {}", jarEntry.getName(),
this.file.getAbsolutePath());
}

final String className = jarEntry.getName().replaceAll("/", "\\.").substring(0,
(jarEntry.getName().length() - 6));
try {
Class<?> clazz = ClassPathLoader.getLatest().forName(className);
Collection<Function> registerableFunctions = getRegisterableFunctionsFromClass(clazz);
for (Function function : registerableFunctions) {
FunctionService.registerFunction(function);
if (isDebugEnabled) {
logger.debug("Registering function class: {}, from JAR file: {}", className,
this.file.getAbsolutePath());

if (functionClasses.contains(className)) {
if (isDebugEnabled) {
logger.debug("Attempting to load class: {}, from JAR file: {}", jarEntry.getName(),
this.file.getAbsolutePath());
}
try {
Class<?> clazz = ClassPathLoader.getLatest().forName(className);
Collection<Function> registerableFunctions = getRegisterableFunctionsFromClass(clazz);
for (Function function : registerableFunctions) {
FunctionService.registerFunction(function);
if (isDebugEnabled) {
logger.debug("Registering function class: {}, from JAR file: {}", className,
this.file.getAbsolutePath());
}
this.registeredFunctions.add(function);
}
this.registeredFunctions.add(function);
} catch (ClassNotFoundException | NoClassDefFoundError cnfex) {
logger.error("Unable to load all classes from JAR file: {}",
this.file.getAbsolutePath(), cnfex);
throw cnfex;
}
} else {
if (isDebugEnabled) {
logger.debug("No functions found in class: {}, from JAR file: {}", jarEntry.getName(),
this.file.getAbsolutePath());
}
} catch (ClassNotFoundException cnfex) {
logger.error("Unable to load all classes from JAR file: {}",
this.file.getAbsolutePath(), cnfex);
throw cnfex;
} catch (NoClassDefFoundError ncdfex) {
logger.error("Unable to load all classes from JAR file: {}",
this.file.getAbsolutePath(), ncdfex);
throw ncdfex;
}
}
jarEntry = jarInputStream.getNextJarEntry();
Expand Down Expand Up @@ -327,6 +320,15 @@ private Collection<Function> getRegisterableFunctionsFromClass(Class<?> clazz) {
return registerableFunctions;
}

private List<String> findFunctionsInThisJar() throws IOException {
URLClassLoader urlClassLoader =
new URLClassLoader(new URL[] {this.getFile().getCanonicalFile().toURL()});
FastClasspathScanner fastClasspathScanner = new FastClasspathScanner()
.removeTemporaryFilesAfterScan(true).overrideClassLoaders(urlClassLoader);
ScanResult scanResult = fastClasspathScanner.scan();
return scanResult.getNamesOfClassesImplementing(Function.class);
}

private Function newFunction(final Class<Function> clazz, final boolean errorOnNoSuchMethod) {
try {
final Constructor<Function> constructor = clazz.getConstructor();
Expand All @@ -342,20 +344,11 @@ private Function newFunction(final Class<Function> clazz, final boolean errorOnN
clazz.getName());
}
}
} catch (SecurityException sex) {
logger.error("Zero-arg constructor of function not accessible for class: {}", clazz.getName(),
sex);
} catch (IllegalAccessException iae) {
logger.error("Zero-arg constructor of function not accessible for class: {}", clazz.getName(),
iae);
} catch (InvocationTargetException ite) {
} catch (Exception ex) {
logger.error("Error when attempting constructor for function for class: {}", clazz.getName(),
ite);
} catch (InstantiationException ie) {
logger.error("Unable to instantiate function for class: {}", clazz.getName(), ie);
} catch (ExceptionInInitializerError eiiex) {
logger.error("Error during function initialization for class: {}", clazz.getName(), eiiex);
ex);
}

return null;
}

Expand Down
7 changes: 2 additions & 5 deletions geode-core/src/main/java/org/apache/geode/internal/JarDeployer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.OutputStream;
import java.io.Serializable;
Expand All @@ -35,7 +34,6 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
Expand All @@ -51,7 +49,6 @@
public class JarDeployer implements Serializable {
private static final long serialVersionUID = 1L;
private static final Logger logger = LogService.getLogger();
public static final String JAR_PREFIX = "";
public static final String JAR_PREFIX_FOR_REGEX = "";
private static final Lock lock = new ReentrantLock();

Expand Down Expand Up @@ -438,7 +435,7 @@ public DeployedJar findLatestValidDeployedJarFromDisk(String unversionedJarName)
Optional<File> latestValidDeployedJarOptional =
Arrays.stream(jarFiles).filter(Objects::nonNull).filter(jarFile -> {
try {
return DeployedJar.isValidJarContent(FileUtils.readFileToByteArray(jarFile));
return DeployedJar.hasValidJarContent(FileUtils.readFileToByteArray(jarFile));
} catch (IOException e) {
return false;
}
Expand Down Expand Up @@ -501,7 +498,7 @@ public List<DeployedJar> deploy(final String jarNames[], final byte[][] jarBytes
DeployedJar[] deployedJars = new DeployedJar[jarNames.length];

for (int i = 0; i < jarNames.length; i++) {
if (!DeployedJar.isValidJarContent(jarBytes[i])) {
if (!DeployedJar.hasValidJarContent(jarBytes[i])) {
throw new IllegalArgumentException(
"File does not contain valid JAR content: " + jarNames[i]);
}
Expand Down
4 changes: 0 additions & 4 deletions geode-core/src/test/java/org/apache/geode/internal/ClassPathLoaderIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,6 @@ public class ClassPathLoaderIntegrationTest {

private File tempFile;
private File tempFile2;
private File extLibsDir;

@Rule
public RestoreSystemProperties restoreSystemProperties = new RestoreSystemProperties();
Expand All @@ -78,9 +77,6 @@ public class ClassPathLoaderIntegrationTest {
public void setUp() throws Exception {
System.setProperty(ClassPathLoader.EXCLUDE_TCCL_PROPERTY, "false");

extLibsDir = new File(this.temporaryFolder.getRoot(), "ext");
extLibsDir.mkdirs();

this.tempFile = this.temporaryFolder.newFile("tempFile1.tmp");
FileOutputStream fos = new FileOutputStream(this.tempFile);
fos.write(new byte[TEMP_FILE_BYTES_COUNT]);
Expand Down
Loading

0 comments on commit 6f7f943

Please sign in to comment.