forked from apache/spark
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[SPARK-3045] Make Serializer interface Java friendly
Author: Reynold Xin <[email protected]> Closes apache#1948 from rxin/kryo and squashes the following commits: a3a80d8 [Reynold Xin] [SPARK-3046] use executor's class loader as the default serializer classloader 3d13277 [Reynold Xin] Reverted that in TestJavaSerializerImpl too. 196f3dc [Reynold Xin] Ok one more commit to revert the classloader change. c49b50c [Reynold Xin] Removed JavaSerializer change. afbf37d [Reynold Xin] Moved the test case also. a2e693e [Reynold Xin] Removed the Kryo bug fix from this pull request. c81bd6c [Reynold Xin] Use defaultClassLoader when executing user specified custom registrator. 68f261e [Reynold Xin] Added license check excludes. 0c28179 [Reynold Xin] [SPARK-3045] Make Serializer interface Java friendly [SPARK-3046] Set executor's class loader as the default serializer class loader
- Loading branch information
Showing
8 changed files
with
193 additions
and
73 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
95 changes: 95 additions & 0 deletions
95
core/src/test/java/org/apache/spark/serializer/TestJavaSerializerImpl.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.spark.serializer; | ||
|
|
||
| import java.io.InputStream; | ||
| import java.io.OutputStream; | ||
| import java.nio.ByteBuffer; | ||
|
|
||
| import scala.Option; | ||
| import scala.reflect.ClassTag; | ||
|
|
||
|
|
||
| /** | ||
| * A simple Serializer implementation to make sure the API is Java-friendly. | ||
| */ | ||
| class TestJavaSerializerImpl extends Serializer { | ||
|
|
||
| @Override | ||
| public SerializerInstance newInstance() { | ||
| return null; | ||
| } | ||
|
|
||
| static class SerializerInstanceImpl extends SerializerInstance { | ||
| @Override | ||
| public <T> ByteBuffer serialize(T t, ClassTag<T> evidence$1) { | ||
| return null; | ||
| } | ||
|
|
||
| @Override | ||
| public <T> T deserialize(ByteBuffer bytes, ClassLoader loader, ClassTag<T> evidence$1) { | ||
| return null; | ||
| } | ||
|
|
||
| @Override | ||
| public <T> T deserialize(ByteBuffer bytes, ClassTag<T> evidence$1) { | ||
| return null; | ||
| } | ||
|
|
||
| @Override | ||
| public SerializationStream serializeStream(OutputStream s) { | ||
| return null; | ||
| } | ||
|
|
||
| @Override | ||
| public DeserializationStream deserializeStream(InputStream s) { | ||
| return null; | ||
| } | ||
| } | ||
|
|
||
| static class SerializationStreamImpl extends SerializationStream { | ||
|
|
||
| @Override | ||
| public <T> SerializationStream writeObject(T t, ClassTag<T> evidence$1) { | ||
| return null; | ||
| } | ||
|
|
||
| @Override | ||
| public void flush() { | ||
|
|
||
| } | ||
|
|
||
| @Override | ||
| public void close() { | ||
|
|
||
| } | ||
| } | ||
|
|
||
| static class DeserializationStreamImpl extends DeserializationStream { | ||
|
|
||
| @Override | ||
| public <T> T readObject(ClassTag<T> evidence$1) { | ||
| return null; | ||
| } | ||
|
|
||
| @Override | ||
| public void close() { | ||
|
|
||
| } | ||
| } | ||
| } |
52 changes: 52 additions & 0 deletions
52
core/src/test/scala/org/apache/spark/serializer/KryoSerializerResizableOutputSuite.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.spark.serializer | ||
|
|
||
| import org.scalatest.FunSuite | ||
|
|
||
| import org.apache.spark.SparkConf | ||
| import org.apache.spark.SparkContext | ||
| import org.apache.spark.LocalSparkContext | ||
| import org.apache.spark.SparkException | ||
|
|
||
|
|
||
| class KryoSerializerResizableOutputSuite extends FunSuite { | ||
|
|
||
| // trial and error showed this will not serialize with 1mb buffer | ||
| val x = (1 to 400000).toArray | ||
|
|
||
| test("kryo without resizable output buffer should fail on large array") { | ||
| val conf = new SparkConf(false) | ||
| conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") | ||
| conf.set("spark.kryoserializer.buffer.mb", "1") | ||
| conf.set("spark.kryoserializer.buffer.max.mb", "1") | ||
| val sc = new SparkContext("local", "test", conf) | ||
| intercept[SparkException](sc.parallelize(x).collect()) | ||
| LocalSparkContext.stop(sc) | ||
| } | ||
|
|
||
| test("kryo with resizable output buffer should succeed on large array") { | ||
| val conf = new SparkConf(false) | ||
| conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") | ||
| conf.set("spark.kryoserializer.buffer.mb", "1") | ||
| conf.set("spark.kryoserializer.buffer.max.mb", "2") | ||
| val sc = new SparkContext("local", "test", conf) | ||
| assert(sc.parallelize(x).collect() === x) | ||
| LocalSparkContext.stop(sc) | ||
| } | ||
| } |
Oops, something went wrong.