ENT-1906: Update DJVM to load deterministic rt.jar into sandbox. (cor…

…da#3973)

* Update DJVM to load deterministic rt.jar into sandbox.
* Disallow invocations of notify(), notifyAll() and wait() APIs.
* Pass entire derivedMember to MemberVisitorImpl.
* Updates after review.
* Refactor MethodBody handlers to use EmitterModule.

djvm/build.gradle

@@ -3,6 +3,7 @@ plugins {
 }
 apply plugin: 'net.corda.plugins.publish-utils'
 apply plugin: 'com.jfrog.artifactory'
+apply plugin: 'idea'
 
 description 'Corda deterministic JVM sandbox'
 
@@ -11,8 +12,18 @@ ext {
     asm_version = '6.1.1'
 }
 
+repositories {
+    maven {
+        url "$artifactory_contextUrl/corda-dev"
+    }
+}
+
 configurations {
     testCompile.extendsFrom shadow
+    jdkRt.resolutionStrategy {
+        // Always check the repository for a newer SNAPSHOT.
+        cacheChangingModulesFor 0, 'seconds'
+    }
 }
 
 dependencies {
@@ -32,6 +43,7 @@ dependencies {
     testCompile "junit:junit:$junit_version"
     testCompile "org.assertj:assertj-core:$assertj_version"
     testCompile "org.apache.logging.log4j:log4j-slf4j-impl:$log4j_version"
+    jdkRt "net.corda:deterministic-rt:latest.integration"
 }
 
 jar.enabled = false
@@ -43,6 +55,10 @@ shadowJar {
 }
 assemble.dependsOn shadowJar
 
+tasks.withType(Test) {
+    systemProperty 'deterministic-rt.path', configurations.jdkRt.asPath
+}
+
 artifacts {
     publish shadowJar
 }
@@ -51,3 +67,10 @@ publish {
     dependenciesFrom configurations.shadow
     name shadowJar.baseName
 }
+
+idea {
+    module {
+        downloadJavadoc = true
+        downloadSources = true
+    }
+}

djvm/cli/src/main/kotlin/net/corda/djvm/tools/cli/ClassCommand.kt

@@ -63,7 +63,7 @@ abstract class ClassCommand : CommandBase() {
 
     private lateinit var classLoader: ClassLoader
 
-    protected var executor = SandboxExecutor<Any, Any>()
+    protected var executor = SandboxExecutor<Any, Any>(SandboxConfiguration.DEFAULT)
 
     private var derivedWhitelist: Whitelist = Whitelist.MINIMAL
 
@@ -114,7 +114,7 @@ abstract class ClassCommand : CommandBase() {
     }
 
     private fun findDiscoverableRunnables(filters: Array<String>): List<Class<*>> {
-        val classes = find<DiscoverableRunnable>()
+        val classes = find<java.util.function.Function<*,*>>()
         val applicableFilters = filters
                 .filter { !isJarFile(it) && !isFullClassName(it) }
         val filteredClasses = applicableFilters
@@ -125,7 +125,7 @@ abstract class ClassCommand : CommandBase() {
                 }
 
         if (applicableFilters.isNotEmpty() && filteredClasses.isEmpty()) {
-            throw Exception("Could not find any classes implementing ${SandboxedRunnable::class.java.simpleName} " +
+            throw Exception("Could not find any classes implementing ${java.util.function.Function::class.java.simpleName} " +
                     "whose name matches '${applicableFilters.joinToString(" ")}'")
         }
 
@@ -189,7 +189,7 @@ abstract class ClassCommand : CommandBase() {
                 profile = profile,
                 rules = if (ignoreRules) { emptyList() } else { Discovery.find() },
                 emitters = ignoreEmitters.emptyListIfTrueOtherwiseNull(),
-                definitionProviders = if(ignoreDefinitionProviders) { emptyList() } else { Discovery.find() },
+                definitionProviders = if (ignoreDefinitionProviders) { emptyList() } else { Discovery.find() },
                 enableTracing = !disableTracing,
                 analysisConfiguration = AnalysisConfiguration(
                         whitelist = whitelist,

djvm/cli/src/main/kotlin/net/corda/djvm/tools/cli/RunCommand.kt

@@ -1,6 +1,5 @@
 package net.corda.djvm.tools.cli
 
-import net.corda.djvm.execution.SandboxedRunnable
 import net.corda.djvm.source.ClassSource
 import picocli.CommandLine.Command
 import picocli.CommandLine.Parameters
@@ -20,7 +19,7 @@ class RunCommand : ClassCommand() {
     var classes: Array<String> = emptyArray()
 
     override fun processClasses(classes: List<Class<*>>) {
-        val interfaceName = SandboxedRunnable::class.java.simpleName
+        val interfaceName = java.util.function.Function::class.java.simpleName
         for (clazz in classes) {
             if (!clazz.interfaces.any { it.simpleName == interfaceName }) {
                 printError("Class is not an instance of $interfaceName; ${clazz.name}")

djvm/cli/src/main/kotlin/net/corda/djvm/tools/cli/WhitelistGenerateCommand.kt

@@ -33,52 +33,53 @@ class WhitelistGenerateCommand : CommandBase() {
     override fun validateArguments() = paths.isNotEmpty()
 
     override fun handleCommand(): Boolean {
-        val entries = mutableListOf<String>()
-        val visitor = object : ClassAndMemberVisitor() {
-            override fun visitClass(clazz: ClassRepresentation): ClassRepresentation {
-                entries.add(clazz.name)
-                return super.visitClass(clazz)
-            }
+        val entries = AnalysisConfiguration().use { configuration ->
+            val entries = mutableListOf<String>()
+            val visitor = object : ClassAndMemberVisitor(configuration, null) {
+                override fun visitClass(clazz: ClassRepresentation): ClassRepresentation {
+                    entries.add(clazz.name)
+                    return super.visitClass(clazz)
+                }
 
-            override fun visitMethod(clazz: ClassRepresentation, method: Member): Member {
-                visitMember(clazz, method)
-                return super.visitMethod(clazz, method)
-            }
+                override fun visitMethod(clazz: ClassRepresentation, method: Member): Member {
+                    visitMember(clazz, method)
+                    return super.visitMethod(clazz, method)
+                }
 
-            override fun visitField(clazz: ClassRepresentation, field: Member): Member {
-                visitMember(clazz, field)
-                return super.visitField(clazz, field)
-            }
+                override fun visitField(clazz: ClassRepresentation, field: Member): Member {
+                    visitMember(clazz, field)
+                    return super.visitField(clazz, field)
+                }
 
-            private fun visitMember(clazz: ClassRepresentation, member: Member) {
-                entries.add("${clazz.name}.${member.memberName}:${member.signature}")
+                private fun visitMember(clazz: ClassRepresentation, member: Member) {
+                    entries.add("${clazz.name}.${member.memberName}:${member.signature}")
+                }
             }
-        }
-        val context = AnalysisContext.fromConfiguration(AnalysisConfiguration(), emptyList())
-        for (path in paths) {
-            ClassSource.fromPath(path).getStreamIterator().forEach {
-                visitor.analyze(it, context)
+            val context = AnalysisContext.fromConfiguration(configuration)
+            for (path in paths) {
+                ClassSource.fromPath(path).getStreamIterator().forEach {
+                    visitor.analyze(it, context)
+                }
             }
+            entries
         }
-        val output = output
-        if (output != null) {
-            Files.newOutputStream(output, StandardOpenOption.CREATE).use {
-                GZIPOutputStream(it).use {
-                    PrintStream(it).use {
-                        it.println("""
+        output?.also {
+            Files.newOutputStream(it, StandardOpenOption.CREATE).use { out ->
+                GZIPOutputStream(out).use { gzip ->
+                    PrintStream(gzip).use { pout ->
+                        pout.println("""
                             |java/.*
                             |javax/.*
                             |jdk/.*
+                            |com/sun/.*
                             |sun/.*
                             |---
                             """.trimMargin().trim())
-                        printEntries(it, entries)
+                        printEntries(pout, entries)
                     }
                 }
             }
-        } else {
-            printEntries(System.out, entries)
-        }
+        } ?: printEntries(System.out, entries)
         return true
     }
 

djvm/src/main/kotlin/net/corda/djvm/SandboxRuntimeContext.kt

@@ -3,25 +3,20 @@ package net.corda.djvm
 import net.corda.djvm.analysis.AnalysisContext
 import net.corda.djvm.costing.RuntimeCostSummary
 import net.corda.djvm.rewiring.SandboxClassLoader
-import net.corda.djvm.source.ClassSource
 
 /**
  * The context in which a sandboxed operation is run.
  *
  * @property configuration The configuration of the sandbox.
- * @property inputClasses The classes passed in for analysis.
  */
-class SandboxRuntimeContext(
-        val configuration: SandboxConfiguration,
-        private val inputClasses: List<ClassSource>
-) {
+class SandboxRuntimeContext(val configuration: SandboxConfiguration) {
 
     /**
      * The class loader to use inside the sandbox.
      */
     val classLoader: SandboxClassLoader = SandboxClassLoader(
             configuration,
-            AnalysisContext.fromConfiguration(configuration.analysisConfiguration, inputClasses)
+            AnalysisContext.fromConfiguration(configuration.analysisConfiguration)
     )
 
     /**
@@ -35,17 +30,15 @@ class SandboxRuntimeContext(
     fun use(action: SandboxRuntimeContext.() -> Unit) {
         SandboxRuntimeContext.instance = this
         try {
-            this.action()
+            action(this)
         } finally {
             threadLocalContext.remove()
         }
     }
 
     companion object {
 
-        private val threadLocalContext = object : ThreadLocal<SandboxRuntimeContext?>() {
-            override fun initialValue(): SandboxRuntimeContext? = null
-        }
+        private val threadLocalContext = ThreadLocal<SandboxRuntimeContext?>()
 
         /**
          * When called from within a sandbox, this returns the context for the current sandbox thread.

djvm/src/main/kotlin/net/corda/djvm/analysis/AnalysisConfiguration.kt

@@ -1,9 +1,15 @@
 package net.corda.djvm.analysis
 
+import net.corda.djvm.code.ruleViolationError
+import net.corda.djvm.code.thresholdViolationError
 import net.corda.djvm.messages.Severity
 import net.corda.djvm.references.ClassModule
 import net.corda.djvm.references.MemberModule
+import net.corda.djvm.source.BootstrapClassLoader
+import net.corda.djvm.source.SourceClassLoader
 import sandbox.net.corda.djvm.costing.RuntimeCostAccounter
+import java.io.Closeable
+import java.io.IOException
 import java.nio.file.Path
 
 /**
@@ -13,7 +19,8 @@ import java.nio.file.Path
  * @param additionalPinnedClasses Classes that have already been declared in the sandbox namespace and that should be
  * made available inside the sandboxed environment.
  * @property minimumSeverityLevel The minimum severity level to log and report.
- * @property classPath The extended class path to use for the analysis.
+ * @param classPath The extended class path to use for the analysis.
+ * @param bootstrapJar The location of a jar containing the Java APIs.
  * @property analyzeAnnotations Analyze annotations despite not being explicitly referenced.
  * @property prefixFilters Only record messages where the originating class name matches one of the provided prefixes.
  * If none are provided, all messages will be reported.
@@ -24,32 +31,47 @@ class AnalysisConfiguration(
         val whitelist: Whitelist = Whitelist.MINIMAL,
         additionalPinnedClasses: Set<String> = emptySet(),
         val minimumSeverityLevel: Severity = Severity.WARNING,
-        val classPath: List<Path> = emptyList(),
+        classPath: List<Path> = emptyList(),
+        bootstrapJar: Path? = null,
         val analyzeAnnotations: Boolean = false,
         val prefixFilters: List<String> = emptyList(),
         val classModule: ClassModule = ClassModule(),
         val memberModule: MemberModule = MemberModule()
-) {
+) : Closeable {
 
     /**
      * Classes that have already been declared in the sandbox namespace and that should be made
      * available inside the sandboxed environment.
      */
-    val pinnedClasses: Set<String> = setOf(SANDBOXED_OBJECT, RUNTIME_COST_ACCOUNTER) + additionalPinnedClasses
+    val pinnedClasses: Set<String> = setOf(
+        SANDBOXED_OBJECT,
+        RuntimeCostAccounter.TYPE_NAME,
+        ruleViolationError,
+        thresholdViolationError
+    ) + additionalPinnedClasses
 
     /**
      * Functionality used to resolve the qualified name and relevant information about a class.
      */
     val classResolver: ClassResolver = ClassResolver(pinnedClasses, whitelist, SANDBOX_PREFIX)
 
+    private val bootstrapClassLoader = bootstrapJar?.let { BootstrapClassLoader(it, classResolver) }
+    val supportingClassLoader = SourceClassLoader(classPath, classResolver, bootstrapClassLoader)
+
+    @Throws(IOException::class)
+    override fun close() {
+        supportingClassLoader.use {
+            bootstrapClassLoader?.close()
+        }
+    }
+
     companion object {
         /**
          * The package name prefix to use for classes loaded into a sandbox.
          */
         private const val SANDBOX_PREFIX: String = "sandbox/"
 
-        private const val SANDBOXED_OBJECT = "sandbox/java/lang/Object"
-        private const val RUNTIME_COST_ACCOUNTER = RuntimeCostAccounter.TYPE_NAME
+        private const val SANDBOXED_OBJECT = SANDBOX_PREFIX + "java/lang/Object"
     }
 
 }

djvm/src/main/kotlin/net/corda/djvm/analysis/AnalysisContext.kt

@@ -1,10 +1,10 @@
 package net.corda.djvm.analysis
 
+import net.corda.djvm.code.asPackagePath
 import net.corda.djvm.messages.MessageCollection
 import net.corda.djvm.references.ClassHierarchy
 import net.corda.djvm.references.EntityReference
 import net.corda.djvm.references.ReferenceMap
-import net.corda.djvm.source.ClassSource
 
 /**
  * The context in which one or more classes are analysed.
@@ -13,13 +13,11 @@ import net.corda.djvm.source.ClassSource
  * @property classes List of class definitions that have been analyzed.
  * @property references A collection of all referenced members found during analysis together with the locations from
  * where each member has been accessed or invoked.
- * @property inputClasses The classes passed in for analysis.
  */
 class AnalysisContext private constructor(
         val messages: MessageCollection,
         val classes: ClassHierarchy,
-        val references: ReferenceMap,
-        val inputClasses: List<ClassSource>
+        val references: ReferenceMap
 ) {
 
     private val origins = mutableMapOf<String, MutableSet<EntityReference>>()
@@ -28,7 +26,7 @@ class AnalysisContext private constructor(
      * Record a class origin in the current analysis context.
      */
     fun recordClassOrigin(name: String, origin: EntityReference) {
-        origins.getOrPut(name.normalize()) { mutableSetOf() }.add(origin)
+        origins.getOrPut(name.asPackagePath) { mutableSetOf() }.add(origin)
     }
 
     /**
@@ -42,20 +40,14 @@ class AnalysisContext private constructor(
         /**
          * Create a new analysis context from provided configuration.
          */
-        fun fromConfiguration(configuration: AnalysisConfiguration, classes: List<ClassSource>): AnalysisContext {
+        fun fromConfiguration(configuration: AnalysisConfiguration): AnalysisContext {
             return AnalysisContext(
                     MessageCollection(configuration.minimumSeverityLevel, configuration.prefixFilters),
                     ClassHierarchy(configuration.classModule, configuration.memberModule),
-                    ReferenceMap(configuration.classModule),
-                    classes
+                    ReferenceMap(configuration.classModule)
             )
         }
 
-        /**
-         * Local extension method for normalizing a class name.
-         */
-        private fun String.normalize() = this.replace("/", ".")
-
     }
 
 }