Skip to content

Commit

Permalink
[FLINK-13451][tests] Remove use of Unsafe.defineClass() from CommonTe…
Browse files Browse the repository at this point in the history 
…stUtils

The method Unsafe.defineClass() is removed in Java 11. To support Java 11, we rework the method
"CommonTestUtils.createClassNotInClassPath()" to use a different mechanism.

This commit now writes the class byte code out to a temporary file and create a new URLClassLoader that
loads the class from that file.  That solution is not a complete drop-in replacement, because it cannot
add the class to an existing class loader, but can only create a new pair of (classloader & new-class-in-that-classloader).
Because of that, the commit also adjusts the existing tests to work with that new mechanism.

This closes apache#9251
  • Loading branch information
@StephanEwen
StephanEwen committed Jul 29, 2019
1 parent 1d81374 commit c5b133e
Show file tree
Hide file tree
Showing 7 changed files with 203 additions and 105 deletions.
23 changes: 8 additions & 15 deletions ...java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerClassLoadingTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,11 @@
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.tuple.Tuple1;
import org.apache.flink.core.testutils.CommonTestUtils;

import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.Serializable;
import java.net.URL;
import java.net.URLClassLoader;

import static org.junit.Assert.fail;

/**
Expand All @@ -39,13 +36,9 @@
*/
public class KryoSerializerClassLoadingTest extends SerializerTestBase<Object> {

/** Class loader for the object that is not in the test class path */
private static final ClassLoader CLASS_LOADER =
new URLClassLoader(new URL[0], KryoSerializerClassLoadingTest.class.getClassLoader());

/** An object that is not in the test class path */
private static final Serializable OBJECT_OUT_OF_CLASSPATH =
CommonTestUtils.createObjectForClassNotInClassPath(CLASS_LOADER);
/** Class loader and object that is not in the test class path. */
private static final CommonTestUtils.ObjectAndClassLoader OUTSIDE_CLASS_LOADING =
CommonTestUtils.createObjectFromNewClassLoader();

// ------------------------------------------------------------------------

Expand All @@ -54,7 +47,7 @@ public class KryoSerializerClassLoadingTest extends SerializerTestBase<Object> {
@Before
public void setupClassLoader() {
originalClassLoader = Thread.currentThread().getContextClassLoader();
Thread.currentThread().setContextClassLoader(CLASS_LOADER);
Thread.currentThread().setContextClassLoader(OUTSIDE_CLASS_LOADING.getClassLoader());
}

@After
Expand All @@ -67,7 +60,7 @@ public void restoreOriginalClassLoader() {
@Test
public void guardTestAssumptions() {
try {
Class.forName(OBJECT_OUT_OF_CLASSPATH.getClass().getName());
Class.forName(OUTSIDE_CLASS_LOADING.getObject().getClass().getName());
fail("This test's assumptions are broken");
}
catch (ClassNotFoundException ignored) {
Expand Down Expand Up @@ -98,11 +91,11 @@ protected Object[] getTestData() {
new Integer(7),

// an object whose class is not on the classpath
OBJECT_OUT_OF_CLASSPATH,
OUTSIDE_CLASS_LOADING.getObject(),

// an object whose class IS on the classpath with a nested object whose class
// is NOT on the classpath
new Tuple1<>(OBJECT_OUT_OF_CLASSPATH)
new Tuple1<>(OUTSIDE_CLASS_LOADING.getObject())
};
}

Expand Down
20 changes: 5 additions & 15 deletions ...est/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshotTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,6 @@
import org.junit.Test;

import java.io.IOException;
import java.net.URL;
import java.net.URLClassLoader;

import static org.apache.flink.api.common.typeutils.TypeSerializerMatchers.isCompatibleAsIs;
import static org.apache.flink.api.common.typeutils.TypeSerializerMatchers.isCompatibleWithReconfiguredSerializer;
Expand Down Expand Up @@ -129,12 +127,13 @@ private static TypeSerializerSnapshot<Animal> kryoSnapshotWithMissingClass() thr
private static byte[] unLoadableSnapshotBytes() throws IOException {
final ClassLoader originalClassLoader = Thread.currentThread().getContextClassLoader();

ClassLoader tempClassLoader =
new URLClassLoader(new URL[0], KryoSerializerSnapshotTest.class.getClassLoader());
final CommonTestUtils.ObjectAndClassLoader outsideClassLoading = CommonTestUtils.createObjectFromNewClassLoader();

try {
Thread.currentThread().setContextClassLoader(tempClassLoader);
Thread.currentThread().setContextClassLoader(outsideClassLoading.getClassLoader());

ExecutionConfig conf = registerClassThatIsNotInClassPath(tempClassLoader);
ExecutionConfig conf = new ExecutionConfig();
conf.registerKryoType(outsideClassLoading.getObject().getClass());

KryoSerializer<Animal> previousSerializer = new KryoSerializer<>(Animal.class, conf);
TypeSerializerSnapshot<Animal> previousSnapshot = previousSerializer.snapshotConfiguration();
Expand All @@ -148,15 +147,6 @@ private static byte[] unLoadableSnapshotBytes() throws IOException {
}
}

private static ExecutionConfig registerClassThatIsNotInClassPath(ClassLoader tempClassLoader) {
Object objectForClassNotInClassPath =
CommonTestUtils.createObjectForClassNotInClassPath(tempClassLoader);

ExecutionConfig conf = new ExecutionConfig();
conf.registerKryoType(objectForClassNotInClassPath.getClass());
return conf;
}

private static TypeSerializerSchemaCompatibility<Animal> resolveKryoCompatibility(ExecutionConfig previous, ExecutionConfig current) {
KryoSerializer<Animal> previousSerializer = new KryoSerializer<>(Animal.class, previous);
TypeSerializerSnapshot<Animal> previousSnapshot = previousSerializer.snapshotConfiguration();
Expand Down
7 changes: 3 additions & 4 deletions ...src/test/java/org/apache/flink/runtime/checkpoint/CheckpointSettingsSerializableTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,6 @@
import javax.annotation.Nonnull;
import java.io.IOException;
import java.io.Serializable;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.Collection;
import java.util.Collections;

Expand All @@ -75,8 +73,9 @@ public class CheckpointSettingsSerializableTest extends TestLogger {

@Test
public void testDeserializationOfUserCodeWithUserClassLoader() throws Exception {
final ClassLoader classLoader = new URLClassLoader(new URL[0], getClass().getClassLoader());
final Serializable outOfClassPath = CommonTestUtils.createObjectForClassNotInClassPath(classLoader);
final CommonTestUtils.ObjectAndClassLoader outsideClassLoading = CommonTestUtils.createObjectFromNewClassLoader();
final ClassLoader classLoader = outsideClassLoading.getClassLoader();
final Serializable outOfClassPath = outsideClassLoading.getObject();

final MasterTriggerRestoreHook.Factory[] hooks = {
new TestFactory(outOfClassPath) };
Expand Down
6 changes: 1 addition & 5 deletions flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ErrorInfoTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,6 @@
import org.junit.Test;

import java.io.Serializable;
import java.net.URL;
import java.net.URLClassLoader;

import static org.junit.Assert.assertEquals;

Expand All @@ -50,10 +48,8 @@ private static final class ExceptionWithCustomClassLoader extends Exception {

private static final long serialVersionUID = 42L;

private static final ClassLoader CUSTOM_LOADER = new URLClassLoader(new URL[0]);

@SuppressWarnings("unused")
private final Serializable outOfClassLoader = CommonTestUtils.createObjectForClassNotInClassPath(CUSTOM_LOADER);
private final Serializable outOfClassLoader = CommonTestUtils.createObjectFromNewClassLoader().getObject();

public ExceptionWithCustomClassLoader() {
super("tada");
Expand Down
22 changes: 8 additions & 14 deletions flink-runtime/src/test/java/org/apache/flink/runtime/state/JavaSerializerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,6 @@

import java.io.File;
import java.io.Serializable;
import java.net.URL;
import java.net.URLClassLoader;

import static org.junit.Assert.*;

Expand All @@ -39,13 +37,9 @@
*/
public class JavaSerializerTest extends SerializerTestBase<Serializable> {

/** Class loader for the object that is not in the test class path */
private static final ClassLoader CLASS_LOADER =
new URLClassLoader(new URL[0], JavaSerializerTest.class.getClassLoader());

/** An object that is not in the test class path */
private static final Serializable OBJECT_OUT_OF_CLASSPATH =
CommonTestUtils.createObjectForClassNotInClassPath(CLASS_LOADER);
/** Class loader and object that is not in the test class path. */
private static final CommonTestUtils.ObjectAndClassLoader OUTSIDE_CLASS_LOADING =
CommonTestUtils.createObjectFromNewClassLoader();

// ------------------------------------------------------------------------

Expand All @@ -54,7 +48,7 @@ public class JavaSerializerTest extends SerializerTestBase<Serializable> {
@Before
public void setupClassLoader() {
originalClassLoader = Thread.currentThread().getContextClassLoader();
Thread.currentThread().setContextClassLoader(CLASS_LOADER);
Thread.currentThread().setContextClassLoader(OUTSIDE_CLASS_LOADING.getClassLoader());
}

@After
Expand All @@ -68,7 +62,7 @@ public void restoreOriginalClassLoader() {
public void guardTest() {
// make sure that this test's assumptions hold
try {
Class.forName(OBJECT_OUT_OF_CLASSPATH.getClass().getName());
Class.forName(OUTSIDE_CLASS_LOADING.getObject().getClass().getName());
fail("Test ineffective: The test class that should not be on the classpath is actually on the classpath.");
} catch (ClassNotFoundException e) {
// expected
Expand All @@ -79,7 +73,7 @@ public void guardTest() {

@Override
protected TypeSerializer<Serializable> createSerializer() {
Thread.currentThread().setContextClassLoader(CLASS_LOADER);
Thread.currentThread().setContextClassLoader(OUTSIDE_CLASS_LOADING.getClassLoader());
return new JavaSerializer<>();
}

Expand All @@ -100,10 +94,10 @@ protected Serializable[] getTestData() {
new File("/some/path/that/I/made/up"),

// an object that is not in the classpath
OBJECT_OUT_OF_CLASSPATH,
OUTSIDE_CLASS_LOADING.getObject(),

// an object that is in the classpath with a nested object not in the classpath
new Tuple1<>(OBJECT_OUT_OF_CLASSPATH)
new Tuple1<>(OUTSIDE_CLASS_LOADING.getObject())
};
}

Expand Down
Loading

0 comments on commit c5b133e

Please sign in to comment.