[CORDA-1778, CORDA-1835]: Decoupled configuration parsing mechanism (c…

…orda#4093)

common/README.md

@@ -0,0 +1,9 @@
+# Common libraries
+
+This directory contains modules representing libraries that are reusable in different areas of Corda.
+
+## Rules of the folder
+
+- No dependencies whatsoever on any modules that are not in this directory (no corda-core, test-utils, etc.).
+- No active components, as in, nothing that has a main function in it.
+- Think carefully before using non-internal packages in these libraries.

common/configuration-parsing/README.md

@@ -0,0 +1,23 @@
+# configuration-parsing
+
+This module provides types and functions to facilitate using Typesafe configuration objects.
+
+## Features
+
+1. A multi-step, structured validation framework for Typesafe configurations, allowing to merge Typesafe and application-level rules.
+2. A parsing framework, allowing to extract domain types from raw configuration objects in a versioned, type-safe fashion.
+3. A configuration description framework, allowing to print the expected schema of a configuration object.
+4. A configuration serialization framework, allowing to output the structure and values of a configuration object, potentially obfuscating sensitive data.
+
+## Concepts
+
+The main idea is to create a `Configuration.Specification` to model the expected structure of a Typesafe configuration.
+The specification is then able to parse, validate, describe and serialize a raw Typesafe configuration.
+
+By using `VersionedConfigurationParser`, it is possible to map specific versions to `Configuration.Specification`s and to parse and validate a raw configuration object based on a version header.
+
+Refer to the following tests to gain an understanding of how the library works:
+
+- net.corda.common.configuration.parsing.internal.versioned.VersionedParsingExampleTest
+- net.corda.common.configuration.parsing.internal.SpecificationTest
+- net.corda.common.configuration.parsing.internal.SchemaTest

common/configuration-parsing/build.gradle

@@ -0,0 +1,25 @@
+apply plugin: 'kotlin'
+
+apply plugin: 'net.corda.plugins.publish-utils'
+apply plugin: 'com.jfrog.artifactory'
+
+dependencies {
+    compile group: "org.jetbrains.kotlin", name: "kotlin-stdlib-jdk8", version: kotlin_version
+    compile group: "org.jetbrains.kotlin", name: "kotlin-reflect", version: kotlin_version
+
+    compile group: "com.typesafe", name: "config", version: typesafe_config_version
+
+    compile project(":common-validation")
+
+    testCompile group: "org.jetbrains.kotlin", name: "kotlin-test", version: kotlin_version
+    testCompile group: "junit", name: "junit", version: junit_version
+    testCompile group: "org.assertj", name: "assertj-core", version: assertj_version
+}
+
+jar {
+    baseName 'common-configuration-parsing'
+}
+
+publish {
+    name jar.baseName
+}

common/configuration-parsing/src/main/kotlin/net/corda/common/configuration/parsing/internal/Properties.kt

@@ -0,0 +1,205 @@
+package net.corda.common.configuration.parsing.internal
+
+import com.typesafe.config.*
+import net.corda.common.validation.internal.Validated
+import net.corda.common.validation.internal.Validated.Companion.invalid
+import net.corda.common.validation.internal.Validated.Companion.valid
+
+internal class LongProperty(key: String, sensitive: Boolean = false) : StandardProperty<Long>(key, Long::class.javaObjectType.simpleName, Config::getLong, Config::getLongList, sensitive) {
+
+    override fun validate(target: Config, options: Configuration.Validation.Options?): Valid<Config> {
+
+        val validated = super.validate(target, options)
+        if (validated.isValid && target.getValue(key).unwrapped().toString().contains(".")) {
+            return invalid(ConfigException.WrongType(target.origin(), key, Long::class.javaObjectType.simpleName, Double::class.javaObjectType.simpleName).toValidationError(key, typeName))
+        }
+        return validated
+    }
+}
+
+internal open class StandardProperty<TYPE>(override val key: String, typeNameArg: String, private val extractSingleValue: (Config, String) -> TYPE, internal val extractListValue: (Config, String) -> List<TYPE>, override val isSensitive: Boolean = false, final override val schema: Configuration.Schema? = null) : Configuration.Property.Definition.Standard<TYPE> {
+
+    override fun valueIn(configuration: Config) = extractSingleValue.invoke(configuration, key)
+
+    override val typeName: String = schema?.let { "#${it.name ?: "Object@$key"}" } ?: typeNameArg
+
+    override fun <MAPPED : Any> mapValid(mappedTypeName: String, convert: (TYPE) -> Valid<MAPPED>): Configuration.Property.Definition.Standard<MAPPED> = FunctionalProperty(this, mappedTypeName, extractListValue, convert)
+
+    override fun optional(defaultValue: TYPE?): Configuration.Property.Definition<TYPE?> = OptionalProperty(this, defaultValue)
+
+    override fun list(): Configuration.Property.Definition.Required<List<TYPE>> = ListProperty(this)
+
+    override fun describe(configuration: Config): ConfigValue {
+
+        if (isSensitive) {
+            return ConfigValueFactory.fromAnyRef(Configuration.Property.Definition.SENSITIVE_DATA_PLACEHOLDER)
+        }
+        return schema?.describe(configuration.getConfig(key)) ?: ConfigValueFactory.fromAnyRef(valueIn(configuration))
+    }
+
+    override val isMandatory = true
+
+    override fun validate(target: Config, options: Configuration.Validation.Options?): Valid<Config> {
+
+        val errors = mutableSetOf<Configuration.Validation.Error>()
+        errors += errorsWhenExtractingValue(target)
+        if (errors.isEmpty()) {
+            schema?.let { nestedSchema ->
+                val nestedConfig: Config? = target.getConfig(key)
+                nestedConfig?.let {
+                    errors += nestedSchema.validate(nestedConfig, options).errors.map { error -> error.withContainingPath(*key.split(".").toTypedArray()) }
+                }
+            }
+        }
+        return Validated.withResult(target, errors)
+    }
+
+    override fun toString() = "\"$key\": \"$typeName\""
+}
+
+private class ListProperty<TYPE>(delegate: StandardProperty<TYPE>) : RequiredDelegatedProperty<List<TYPE>, StandardProperty<TYPE>>(delegate) {
+
+    override val typeName: String = "List<${delegate.typeName}>"
+
+    override fun valueIn(configuration: Config): List<TYPE> = delegate.extractListValue.invoke(configuration, key)
+
+    override fun validate(target: Config, options: Configuration.Validation.Options?): Valid<Config> {
+
+        val errors = mutableSetOf<Configuration.Validation.Error>()
+        errors += errorsWhenExtractingValue(target)
+        if (errors.isEmpty()) {
+            delegate.schema?.let { schema ->
+                errors += valueIn(target).asSequence().map { element -> element as ConfigObject }.map(ConfigObject::toConfig).mapIndexed { index, targetConfig -> schema.validate(targetConfig, options).errors.map { error -> error.withContainingPath(key, "[$index]") } }.reduce { one, other -> one + other }
+            }
+        }
+        return Validated.withResult(target, errors)
+    }
+
+    override fun describe(configuration: Config): ConfigValue {
+
+        if (isSensitive) {
+            return ConfigValueFactory.fromAnyRef(Configuration.Property.Definition.SENSITIVE_DATA_PLACEHOLDER)
+        }
+        return delegate.schema?.let { schema -> ConfigValueFactory.fromAnyRef(valueIn(configuration).asSequence().map { element -> element as ConfigObject }.map(ConfigObject::toConfig).map { schema.describe(it) }.toList()) } ?: ConfigValueFactory.fromAnyRef(valueIn(configuration))
+    }
+}
+
+private class OptionalProperty<TYPE>(delegate: Configuration.Property.Definition.Required<TYPE>, private val defaultValue: TYPE?) : DelegatedProperty<TYPE?, Configuration.Property.Definition.Required<TYPE>>(delegate) {
+
+    override val isMandatory: Boolean = false
+
+    override val typeName: String = "${super.typeName}?"
+
+    override fun describe(configuration: Config) = delegate.describe(configuration)
+
+    override fun valueIn(configuration: Config): TYPE? {
+
+        return when {
+            isSpecifiedBy(configuration) -> delegate.valueIn(configuration)
+            else -> defaultValue
+        }
+    }
+
+    override fun validate(target: Config, options: Configuration.Validation.Options?): Valid<Config> {
+
+        val result = delegate.validate(target, options)
+        val error = result.errors.asSequence().filterIsInstance<Configuration.Validation.Error.MissingValue>().singleOrNull()
+        return when {
+            error != null -> if (result.errors.size > 1) result else valid(target)
+            else -> result
+        }
+    }
+}
+
+private class FunctionalProperty<TYPE, MAPPED : Any>(delegate: Configuration.Property.Definition.Standard<TYPE>, private val mappedTypeName: String, internal val extractListValue: (Config, String) -> List<TYPE>, private val convert: (TYPE) -> Valid<MAPPED>) : RequiredDelegatedProperty<MAPPED, Configuration.Property.Definition.Standard<TYPE>>(delegate), Configuration.Property.Definition.Standard<MAPPED> {
+
+    override fun valueIn(configuration: Config) = convert.invoke(delegate.valueIn(configuration)).valueOrThrow()
+
+    override val typeName: String = if (super.typeName == "#$mappedTypeName") super.typeName else "$mappedTypeName(${super.typeName})"
+
+    override fun <M : Any> mapValid(mappedTypeName: String, convert: (MAPPED) -> Valid<M>): Configuration.Property.Definition.Standard<M> = FunctionalProperty(delegate, mappedTypeName, extractListValue, { target: TYPE -> this.convert.invoke(target).mapValid(convert) })
+
+    override fun list(): Configuration.Property.Definition.Required<List<MAPPED>> = FunctionalListProperty(this)
+
+    override fun validate(target: Config, options: Configuration.Validation.Options?): Valid<Config> {
+
+        val errors = mutableSetOf<Configuration.Validation.Error>()
+        errors += delegate.validate(target, options).errors
+        if (errors.isEmpty()) {
+            errors += convert.invoke(delegate.valueIn(target)).mapErrors { error -> error.with(delegate.key, mappedTypeName) }.errors
+        }
+        return Validated.withResult(target, errors)
+    }
+
+    override fun describe(configuration: Config) = delegate.describe(configuration)
+}
+
+private class FunctionalListProperty<RAW, TYPE : Any>(delegate: FunctionalProperty<RAW, TYPE>) : RequiredDelegatedProperty<List<TYPE>, FunctionalProperty<RAW, TYPE>>(delegate) {
+
+    override val typeName: String = "List<${super.typeName}>"
+
+    override fun valueIn(configuration: Config): List<TYPE> = delegate.extractListValue.invoke(configuration, key).asSequence().map { configObject(key to ConfigValueFactory.fromAnyRef(it)) }.map(ConfigObject::toConfig).map(delegate::valueIn).toList()
+
+    override fun validate(target: Config, options: Configuration.Validation.Options?): Valid<Config> {
+
+        val list = try {
+            delegate.extractListValue.invoke(target, key)
+        } catch (e: ConfigException) {
+            if (isErrorExpected(e)) {
+                return invalid(e.toValidationError(key, typeName))
+            } else {
+                throw e
+            }
+        }
+        val errors = list.asSequence().map { configObject(key to ConfigValueFactory.fromAnyRef(it)) }.mapIndexed { index, value -> delegate.validate(value.toConfig(), options).errors.map { error -> error.withContainingPath(key, "[$index]") } }.reduce { one, other -> one + other }.toSet()
+        return Validated.withResult(target, errors)
+    }
+
+    override fun describe(configuration: Config): ConfigValue {
+
+        if (isSensitive) {
+            return ConfigValueFactory.fromAnyRef(Configuration.Property.Definition.SENSITIVE_DATA_PLACEHOLDER)
+        }
+        return delegate.schema?.let { schema -> ConfigValueFactory.fromAnyRef(valueIn(configuration).asSequence().map { element -> element as ConfigObject }.map(ConfigObject::toConfig).map { schema.describe(it) }.toList()) } ?: ConfigValueFactory.fromAnyRef(valueIn(configuration))
+    }
+}
+
+private abstract class DelegatedProperty<TYPE, DELEGATE : Configuration.Property.Metadata>(protected val delegate: DELEGATE) : Configuration.Property.Metadata by delegate, Configuration.Property.Definition<TYPE> {
+
+    final override fun toString() = "\"$key\": \"$typeName\""
+}
+
+private abstract class RequiredDelegatedProperty<TYPE, DELEGATE : Configuration.Property.Definition.Required<*>>(delegate: DELEGATE) : DelegatedProperty<TYPE, DELEGATE>(delegate), Configuration.Property.Definition.Required<TYPE> {
+
+    final override fun optional(defaultValue: TYPE?): Configuration.Property.Definition<TYPE?> = OptionalProperty(this, defaultValue)
+}
+
+private fun ConfigException.toValidationError(keyName: String, typeName: String): Configuration.Validation.Error {
+
+    val toError = when (this) {
+        is ConfigException.Missing -> Configuration.Validation.Error.MissingValue.Companion::of
+        is ConfigException.WrongType -> Configuration.Validation.Error.WrongType.Companion::of
+        is ConfigException.BadValue -> Configuration.Validation.Error.BadValue.Companion::of
+        is ConfigException.BadPath -> Configuration.Validation.Error.BadPath.Companion::of
+        is ConfigException.Parse -> Configuration.Validation.Error.MalformedStructure.Companion::of
+        else -> throw IllegalStateException("Unsupported ConfigException of type ${this::class.java.name}", this)
+    }
+    return toError.invoke(message!!, keyName, typeName, emptyList())
+}
+
+private fun Configuration.Property.Definition<*>.errorsWhenExtractingValue(target: Config): Set<Configuration.Validation.Error> {
+
+    try {
+        valueIn(target)
+        return emptySet()
+    } catch (exception: ConfigException) {
+        if (isErrorExpected(exception)) {
+            return setOf(exception.toValidationError(key, typeName))
+        }
+        throw exception
+    }
+}
+
+private val expectedExceptionTypes = setOf(ConfigException.Missing::class, ConfigException.WrongType::class, ConfigException.BadValue::class, ConfigException.BadPath::class, ConfigException.Parse::class)
+
+private fun isErrorExpected(error: ConfigException) = expectedExceptionTypes.any { expected -> expected.isInstance(error) }

common/configuration-parsing/src/main/kotlin/net/corda/common/configuration/parsing/internal/Schema.kt

@@ -0,0 +1,75 @@
+package net.corda.common.configuration.parsing.internal
+
+import com.typesafe.config.Config
+import com.typesafe.config.ConfigValue
+import com.typesafe.config.ConfigValueFactory
+import net.corda.common.validation.internal.Validated
+
+internal class Schema(override val name: String?, unorderedProperties: Iterable<Configuration.Property.Definition<*>>) : Configuration.Schema {
+
+    override val properties = unorderedProperties.sortedBy(Configuration.Property.Definition<*>::key).toSet()
+
+    init {
+        val invalid = properties.groupBy(Configuration.Property.Definition<*>::key).mapValues { entry -> entry.value.size }.filterValues { propertiesForKey -> propertiesForKey > 1 }
+        if (invalid.isNotEmpty()) {
+            throw IllegalArgumentException("More than one property was found for keys ${invalid.keys.joinToString(", ", "[", "]")}.")
+        }
+    }
+
+    override fun validate(target: Config, options: Configuration.Validation.Options?): Valid<Config> {
+
+        val propertyErrors = properties.flatMap { property -> property.validate(target, options).errors }.toMutableSet()
+        if (options?.strict == true) {
+            val unknownKeys = target.root().keys - properties.map(Configuration.Property.Definition<*>::key)
+            propertyErrors += unknownKeys.map { Configuration.Validation.Error.Unknown.of(it) }
+        }
+        return Validated.withResult(target, propertyErrors)
+    }
+
+    override fun description(): String {
+
+        val description = StringBuilder()
+        val root = properties.asSequence().map { it.key to ConfigValueFactory.fromAnyRef(it.typeName) }.fold(configObject()) { config, (key, value) -> config.withValue(key, value) }
+
+        description.append(root.toConfig().serialize())
+
+        val nestedProperties = (properties + properties.flatMap { it.schema?.properties ?: emptySet() }).asSequence().distinctBy(Configuration.Property.Definition<*>::schema)
+        nestedProperties.forEach { property ->
+            property.schema?.let {
+                description.append(System.lineSeparator())
+                description.append("${property.typeName}: ")
+                description.append(it.description())
+                description.append(System.lineSeparator())
+            }
+        }
+        return description.toString()
+    }
+
+    override fun describe(configuration: Config): ConfigValue {
+
+        return properties.asSequence().map { it.key to it.describe(configuration) }.fold(configObject()) { config, (key, value) -> config.withValue(key, value) }
+    }
+
+    override fun equals(other: Any?): Boolean {
+
+        if (this === other) {
+            return true
+        }
+        if (javaClass != other?.javaClass) {
+            return false
+        }
+
+        other as Schema
+
+        if (properties != other.properties) {
+            return false
+        }
+
+        return true
+    }
+
+    override fun hashCode(): Int {
+
+        return properties.hashCode()
+    }
+}