Merge pull request corda#2251 from corda/kat/feature/testingDisableEv…

…olver

Disable Evolver / Generics fingerprinting bugfix

.gitignore

@@ -88,6 +88,7 @@ crashlytics-build.properties
 
 # docs related
 docs/virtualenv/
+virtualenv/
 
 # bft-smart
 **/config/currentView

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializationInput.kt

@@ -54,7 +54,6 @@ class DeserializationInput(internal val serializerFactory: SerializerFactory) {
     inline fun <reified T : Any> deserialize(bytes: SerializedBytes<T>): T =
             deserialize(bytes, T::class.java)
 
-
     @Throws(NotSerializableException::class)
     inline internal fun <reified T : Any> deserializeAndReturnEnvelope(bytes: SerializedBytes<T>): ObjectAndEnvelope<T> =
             deserializeAndReturnEnvelope(bytes, T::class.java)

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumEvolutionSerializer.kt

@@ -106,7 +106,7 @@ class EnumEvolutionSerializer(
             // to the name as it exists. We want to test any new constants have been added to the end
             // of the enum class
             val serialisedOrds = ((schemas.schema.types.find { it.name == old.name } as RestrictedType).choices
-                    .associateBy ({ it.value.toInt() }, { conversions[it.name] }))
+                    .associateBy({ it.value.toInt() }, { conversions[it.name] }))
 
             if (ordinals.filterNot { serialisedOrds[it.value] == it.key }.isNotEmpty()) {
                 throw NotSerializableException("Constants have been reordered, additions must be appended to the end")
@@ -133,4 +133,4 @@ class EnumEvolutionSerializer(
     override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
         throw UnsupportedOperationException("It should be impossible to write an evolution serializer")
     }
-}
+}

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt

@@ -130,3 +130,34 @@ class EvolutionSerializer(
     }
 }
 
+/**
+ * Instances of this type are injected into a [SerializerFactory] at creation time to dictate the
+ * behaviour of evolution within that factory. Under normal circumstances this will simply
+ * be an object that returns an [EvolutionSerializer]. Of course, any implementation that
+ * extends this class can be written to invoke whatever behaviour is desired.
+ */
+abstract class EvolutionSerializerGetterBase {
+    abstract fun getEvolutionSerializer(
+            factory: SerializerFactory,
+            typeNotation: TypeNotation,
+            newSerializer: AMQPSerializer<Any>,
+            schemas: SerializationSchemas): AMQPSerializer<Any>
+}
+
+/**
+ * The normal use case for generating an [EvolutionSerializer]'s based on the differences
+ * between the received schema and the class as it exists now on the class path,
+ */
+class EvolutionSerializerGetter : EvolutionSerializerGetterBase() {
+    override fun getEvolutionSerializer(factory: SerializerFactory,
+                                        typeNotation: TypeNotation,
+                                        newSerializer: AMQPSerializer<Any>,
+                                        schemas: SerializationSchemas): AMQPSerializer<Any> =
+            factory.getSerializersByDescriptor().computeIfAbsent(typeNotation.descriptor.name!!) {
+                when (typeNotation) {
+                    is CompositeType -> EvolutionSerializer.make(typeNotation, newSerializer as ObjectSerializer, factory)
+                    is RestrictedType -> EnumEvolutionSerializer.make(typeNotation, newSerializer, factory, schemas)
+                }
+            }
+}
+

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Schema.kt

@@ -346,19 +346,54 @@ private fun Hasher.fingerprintWithCustomSerializerOrElse(factory: SerializerFact
     }
 }
 
-// This method concatentates various elements of the types recursively as unencoded strings into the hasher, effectively
+// This method concatenates various elements of the types recursively as unencoded strings into the hasher, effectively
 // creating a unique string for a type which we then hash in the calling function above.
-private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: MutableSet<Type>, hasher: Hasher, factory: SerializerFactory): Hasher {
-    return if (type in alreadySeen) {
+private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: MutableSet<Type>,
+                               hasher: Hasher, factory: SerializerFactory, offset: Int = 4): Hasher {
+
+    // We don't include Example<?> and Example<T> where type is ? or T in this otherwise we
+    // generate different fingerprints for class Outer<T>(val a: Inner<T>) when serialising
+    // and deserializing (assuming deserialization is occurring in a factory that didn't
+    // serialise the object in the  first place (and thus the cache lookup fails). This is also
+    // true of Any, where we need  Example<A, B> and Example<?, ?> to have the same fingerprint
+    return if (type in alreadySeen && (type !is SerializerFactory.AnyType) && (type !is TypeVariable<*>)) {
         hasher.putUnencodedChars(ALREADY_SEEN_HASH)
     } else {
         alreadySeen += type
         try {
             when (type) {
-                is SerializerFactory.AnyType -> hasher.putUnencodedChars(ANY_TYPE_HASH)
+                is ParameterizedType -> {
+                    // Hash the rawType + params
+                    val clazz = type.rawType as Class<*>
+
+                    val startingHash = if (isCollectionOrMap(clazz)) {
+                        hasher.putUnencodedChars(clazz.name)
+                    } else {
+                        hasher.fingerprintWithCustomSerializerOrElse(factory, clazz, type) {
+                            fingerprintForObject(type, type, alreadySeen, hasher, factory, offset+4)
+                        }
+                    }
+
+                    // ... and concatenate the type data for each parameter type.
+                    type.actualTypeArguments.fold(startingHash) { orig, paramType ->
+                        fingerprintForType(paramType, type, alreadySeen, orig, factory, offset+4)
+                    }
+                }
+            // Treat generic types as "any type" to prevent fingerprint mismatch. This case we fall into when
+            // looking at A and B from Example<A, B> (remember we call this function recursively). When
+            // serialising a concrete example of the type we have A and B which are TypeVariables<*>'s but
+            // when deserializing we only have the wildcard placeholder ?, or AnyType
+            //
+            // Note, TypeVariable<*> used to be encoded as TYPE_VARIABLE_HASH but that again produces a
+            // differing fingerprint on serialisation and deserialization
+                is SerializerFactory.AnyType,
+                is TypeVariable<*> -> {
+                    hasher.putUnencodedChars("?").putUnencodedChars(ANY_TYPE_HASH)
+                }
                 is Class<*> -> {
                     if (type.isArray) {
-                        fingerprintForType(type.componentType, contextType, alreadySeen, hasher, factory).putUnencodedChars(ARRAY_HASH)
+                        fingerprintForType(type.componentType, contextType, alreadySeen, hasher, factory, offset+4)
+                                .putUnencodedChars(ARRAY_HASH)
                     } else if (SerializerFactory.isPrimitive(type)) {
                         hasher.putUnencodedChars(type.name)
                     } else if (isCollectionOrMap(type)) {
@@ -377,32 +412,18 @@ private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: Muta
                                 // to the CorDapp but maybe reference to the JAR in the short term.
                                 hasher.putUnencodedChars(type.name)
                             } else {
-                                fingerprintForObject(type, type, alreadySeen, hasher, factory)
+                                fingerprintForObject(type, type, alreadySeen, hasher, factory, offset+4)
                             }
                         }
                     }
                 }
-                is ParameterizedType -> {
-                    // Hash the rawType + params
-                    val clazz = type.rawType as Class<*>
-                    val startingHash = if (isCollectionOrMap(clazz)) {
-                        hasher.putUnencodedChars(clazz.name)
-                    } else {
-                        hasher.fingerprintWithCustomSerializerOrElse(factory, clazz, type) {
-                            fingerprintForObject(type, type, alreadySeen, hasher, factory)
-                        }
-                    }
-                    // ... and concatentate the type data for each parameter type.
-                    type.actualTypeArguments.fold(startingHash) { orig, paramType ->
-                        fingerprintForType(paramType, type, alreadySeen, orig, factory)
-                    }
-                }
             // Hash the element type + some array hash
                 is GenericArrayType -> fingerprintForType(type.genericComponentType, contextType, alreadySeen,
-                        hasher, factory).putUnencodedChars(ARRAY_HASH)
+                        hasher, factory, offset+4).putUnencodedChars(ARRAY_HASH)
             // TODO: include bounds
-                is TypeVariable<*> -> hasher.putUnencodedChars(type.name).putUnencodedChars(TYPE_VARIABLE_HASH)
-                is WildcardType -> hasher.putUnencodedChars(type.typeName).putUnencodedChars(WILDCARD_TYPE_HASH)
+                is WildcardType -> {
+                    hasher.putUnencodedChars(type.typeName).putUnencodedChars(WILDCARD_TYPE_HASH)
+                }
                 else -> throw NotSerializableException("Don't know how to hash")
             }
         } catch (e: NotSerializableException) {
@@ -416,15 +437,21 @@ private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: Muta
 private fun isCollectionOrMap(type: Class<*>) = (Collection::class.java.isAssignableFrom(type) || Map::class.java.isAssignableFrom(type)) &&
         !EnumSet::class.java.isAssignableFrom(type)
 
-private fun fingerprintForObject(type: Type, contextType: Type?, alreadySeen: MutableSet<Type>, hasher: Hasher, factory: SerializerFactory): Hasher {
+private fun fingerprintForObject(
+        type: Type,
+        contextType: Type?,
+        alreadySeen: MutableSet<Type>,
+        hasher: Hasher,
+        factory: SerializerFactory,
+        offset: Int = 0): Hasher {
     // Hash the class + properties + interfaces
     val name = type.asClass()?.name ?: throw NotSerializableException("Expected only Class or ParameterizedType but found $type")
     propertiesForSerialization(constructorForDeserialization(type), contextType ?: type, factory).getters
             .fold(hasher.putUnencodedChars(name)) { orig, prop ->
-                fingerprintForType(prop.resolvedType, type, alreadySeen, orig, factory)
+                fingerprintForType(prop.resolvedType, type, alreadySeen, orig, factory, offset+4)
                         .putUnencodedChars(prop.name)
                         .putUnencodedChars(if (prop.mandatory) NOT_NULLABLE_HASH else NULLABLE_HASH)
             }
-    interfacesForSerialization(type, factory).map { fingerprintForType(it, type, alreadySeen, hasher, factory) }
+    interfacesForSerialization(type, factory).map { fingerprintForType(it, type, alreadySeen, hasher, factory, offset+4) }
     return hasher
 }

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt

@@ -20,6 +20,10 @@ data class FactorySchemaAndDescriptor(val schemas: SerializationSchemas, val typ
 
 /**
  * Factory of serializers designed to be shared across threads and invocations.
+ *
+ * @property evolutionSerializerGetter controls how evolution serializers are generated by the factory. The normal
+ * use case is an [EvolutionSerializer] type is returned. However, in some scenarios, primarily testing, this
+ * can be altered to fit the requirements of the test.
  */
 // TODO: support for intern-ing of deserialized objects for some core types (e.g. PublicKey) for memory efficiency
 // TODO: maybe support for caching of serialized form of some core types for performance
@@ -33,28 +37,27 @@ data class FactorySchemaAndDescriptor(val schemas: SerializationSchemas, val typ
 // TODO: need to rethink matching of constructor to properties in relation to implementing interfaces and needing those properties etc.
 // TODO: need to support super classes as well as interfaces with our current code base... what's involved?  If we continue to ban, what is the impact?
 @ThreadSafe
-open class SerializerFactory(val whitelist: ClassWhitelist, cl: ClassLoader) {
+open class SerializerFactory(
+        val whitelist: ClassWhitelist,
+        cl: ClassLoader,
+        private val evolutionSerializerGetter: EvolutionSerializerGetterBase = EvolutionSerializerGetter()) {
     private val serializersByType = ConcurrentHashMap<Type, AMQPSerializer<Any>>()
     private val serializersByDescriptor = ConcurrentHashMap<Any, AMQPSerializer<Any>>()
     private val customSerializers = CopyOnWriteArrayList<SerializerFor>()
-    val transformsCache = ConcurrentHashMap<String, EnumMap<TransformTypes, MutableList<Transform>>>()
+    private val transformsCache = ConcurrentHashMap<String, EnumMap<TransformTypes, MutableList<Transform>>>()
 
     open val classCarpenter = ClassCarpenter(cl, whitelist)
 
     val classloader: ClassLoader
         get() = classCarpenter.classloader
 
-    private fun getEvolutionSerializer(
-            typeNotation: TypeNotation,
-            newSerializer: AMQPSerializer<Any>,
-            schemas: SerializationSchemas): AMQPSerializer<Any> {
-        return serializersByDescriptor.computeIfAbsent(typeNotation.descriptor.name!!) {
-            when (typeNotation) {
-                is CompositeType -> EvolutionSerializer.make(typeNotation, newSerializer as ObjectSerializer, this)
-                is RestrictedType -> EnumEvolutionSerializer.make(typeNotation, newSerializer, this, schemas)
-            }
-        }
-    }
+    private fun getEvolutionSerializer(typeNotation: TypeNotation, newSerializer: AMQPSerializer<Any>,
+                                       schemas: SerializationSchemas)
+            = evolutionSerializerGetter.getEvolutionSerializer(this, typeNotation, newSerializer, schemas)
+
+    fun getSerializersByDescriptor() = serializersByDescriptor
+
+    fun getTransformsCache() = transformsCache
 
     /**
      * Look up, and manufacture if necessary, a serializer for the given type.
@@ -93,7 +96,9 @@ open class SerializerFactory(val whitelist: ClassWhitelist, cl: ClassLoader) {
                 whitelist.requireWhitelisted(actualType)
                 EnumSerializer(actualType, actualClass ?: declaredClass, this)
             }
-            else -> makeClassSerializer(actualClass ?: declaredClass, actualType, declaredType)
+            else -> {
+                makeClassSerializer(actualClass ?: declaredClass, actualType, declaredType)
+            }
         }
 
         serializersByDescriptor.putIfAbsent(serializer.typeDescriptor, serializer)
@@ -102,23 +107,23 @@ open class SerializerFactory(val whitelist: ClassWhitelist, cl: ClassLoader) {
     }
 
     /**
-     * Try and infer concrete types for any generics type variables for the actual class encountered, based on the declared
-     * type.
+     * Try and infer concrete types for any generics type variables for the actual class encountered,
+     * based on the declared type.
      */
     // TODO: test GenericArrayType
-    private fun inferTypeVariables(actualClass: Class<*>?, declaredClass: Class<*>, declaredType: Type): Type? =
-            when (declaredType) {
-                is ParameterizedType -> inferTypeVariables(actualClass, declaredClass, declaredType)
-            // Nothing to infer, otherwise we'd have ParameterizedType
-                is Class<*> -> actualClass
-                is GenericArrayType -> {
-                    val declaredComponent = declaredType.genericComponentType
-                    inferTypeVariables(actualClass?.componentType, declaredComponent.asClass()!!, declaredComponent)?.asArray()
-                }
-                is TypeVariable<*> -> actualClass
-                is WildcardType -> actualClass
-                else -> null
-            }
+    private fun inferTypeVariables(actualClass: Class<*>?, declaredClass: Class<*>,
+                                   declaredType: Type) : Type? = when (declaredType) {
+        is ParameterizedType -> inferTypeVariables(actualClass, declaredClass, declaredType)
+    // Nothing to infer, otherwise we'd have ParameterizedType
+        is Class<*> -> actualClass
+        is GenericArrayType -> {
+            val declaredComponent = declaredType.genericComponentType
+            inferTypeVariables(actualClass?.componentType, declaredComponent.asClass()!!, declaredComponent)?.asArray()
+        }
+        is TypeVariable<*> -> actualClass
+        is WildcardType -> actualClass
+        else -> null
+    }
 
     /**
      * Try and infer concrete types for any generics type variables for the actual class encountered, based on the declared
@@ -214,9 +219,9 @@ open class SerializerFactory(val whitelist: ClassWhitelist, cl: ClassLoader) {
             try {
                 val serialiser = processSchemaEntry(typeNotation)
 
-                // if we just successfully built a serialiser for the type but the type fingerprint
+                // if we just successfully built a serializer for the type but the type fingerprint
                 // doesn't match that of the serialised object then we are dealing with  different
-                // instance of the class, as such we need to build an EvolutionSerialiser
+                // instance of the class, as such we need to build an EvolutionSerializer
                 if (serialiser.typeDescriptor != typeNotation.descriptor.name) {
                     getEvolutionSerializer(typeNotation, serialiser, schemaAndDescriptor.schemas)
                 }
@@ -337,7 +342,9 @@ open class SerializerFactory(val whitelist: ClassWhitelist, cl: ClassLoader) {
                     "${nameForType(type.componentType)}${if (type.componentType.isPrimitive) "[p]" else "[]"}"
                 } else type.name
             }
-            is ParameterizedType -> "${nameForType(type.rawType)}<${type.actualTypeArguments.joinToString { nameForType(it) }}>"
+            is ParameterizedType -> {
+                "${nameForType(type.rawType)}<${type.actualTypeArguments.joinToString { nameForType(it) }}>"
+            }
             is GenericArrayType -> "${nameForType(type.genericComponentType)}[]"
             is WildcardType -> "?"
             is TypeVariable<*> -> "?"

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/TransformsSchema.kt

@@ -200,7 +200,7 @@ data class TransformsSchema(val types: Map<String, EnumMap<TransformTypes, Mutab
          * @param sf the [SerializerFactory] building this transform set. Needed as each can define it's own
          * class loader and this dictates which classes we can and cannot see
          */
-        fun get(name: String, sf: SerializerFactory) = sf.transformsCache.computeIfAbsent(name) {
+        fun get(name: String, sf: SerializerFactory) = sf.getTransformsCache().computeIfAbsent(name) {
             val transforms = EnumMap<TransformTypes, MutableList<Transform>>(TransformTypes::class.java)
             try {
                 val clazz = sf.classloader.loadClass(name)

node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/ErrorMessageTests.java

@@ -31,7 +31,12 @@ public C(Integer a) {
 
     @Test
     public void testJavaConstructorAnnotations() {
-        SerializerFactory factory1 = new SerializerFactory(AllWhitelist.INSTANCE, ClassLoader.getSystemClassLoader());
+        EvolutionSerializerGetterBase evolutionSerialiserGetter = new EvolutionSerializerGetter();
+        SerializerFactory factory1 = new SerializerFactory(
+                AllWhitelist.INSTANCE,
+                ClassLoader.getSystemClassLoader(),
+                evolutionSerialiserGetter);
+
         SerializationOutput ser = new SerializationOutput(factory1);
 
         Assertions.assertThatThrownBy(() -> ser.serialize(new C(1)))