[CORDA-792] Standalone Shell (corda#2663)

- Existing embedded Shell connects via RPC including checking RPC user credentials (before was a direct use of CordaRPCOps):  in dev mode when console terminal is enabled, node created `shell` user.
- New Standalone Shell app with the same functionalities as Shell: connects to a node via RPC Client,  can use SSL and run SSH server.

CONTRIBUTORS.md

@@ -11,6 +11,7 @@ changes to this list.
 * Andras Slemmer (R3)
 * Andrius Dagys (R3)
 * Andrzej Cichocki (R3)
+* Andrzej Grzesik (R3)
 * Anthony Coates (Deutsche Bank)
 * Anton Semenov (Commerzbank)
 * Antonio Cerrato (SEB)
@@ -92,7 +93,7 @@ changes to this list.
 * Matthijs van den Bos (ING)
 * Michal Kit (R3)
 * Micheal Hinstridge (Thoughtworks)
-* Michelle Sollecito (R3)
+* Michele Sollecito (R3)
 * Mike Hearn (R3)
 * Mike Reichelt (US Bank)
 * Mustafa Ozturk (Natixis)

build.gradle

@@ -73,6 +73,7 @@ buildscript {
     ext.selenium_version = '3.8.1'
     ext.ghostdriver_version = '2.1.0'
     ext.eaagentloader_version = '1.0.3'
+    ext.jsch_version = '0.1.54'
 
     // Update 121 is required for ObjectInputFilter and at time of writing 131 was latest:
     ext.java8_minUpdateVersion = '131'

client/rpc/src/main/kotlin/net/corda/client/rpc/CordaRPCClient.kt

@@ -73,7 +73,8 @@ data class CordaRPCClientConfiguration(val connectionMaxRetryInterval: Duration)
 class CordaRPCClient private constructor(
         hostAndPort: NetworkHostAndPort,
         configuration: CordaRPCClientConfiguration = CordaRPCClientConfiguration.DEFAULT,
-        sslConfiguration: SSLConfiguration? = null
+        sslConfiguration: SSLConfiguration? = null,
+        classLoader: ClassLoader? = null
 ) {
     @JvmOverloads
     constructor(hostAndPort: NetworkHostAndPort, configuration: CordaRPCClientConfiguration = CordaRPCClientConfiguration.DEFAULT) : this(hostAndPort, configuration, null)
@@ -86,14 +87,23 @@ class CordaRPCClient private constructor(
         ): CordaRPCClient {
             return CordaRPCClient(hostAndPort, configuration, sslConfiguration)
         }
+
+        internal fun createWithSslAndClassLoader(
+                hostAndPort: NetworkHostAndPort,
+                configuration: CordaRPCClientConfiguration = CordaRPCClientConfiguration.DEFAULT,
+                sslConfiguration: SSLConfiguration? = null,
+                classLoader: ClassLoader? = null
+        ): CordaRPCClient {
+            return CordaRPCClient(hostAndPort, configuration, sslConfiguration, classLoader)
+        }
     }
 
     init {
         try {
             effectiveSerializationEnv
         } catch (e: IllegalStateException) {
             try {
-                KryoClientSerializationScheme.initialiseSerialization()
+                KryoClientSerializationScheme.initialiseSerialization(classLoader)
             } catch (e: IllegalStateException) {
                 // Race e.g. two of these constructed in parallel, ignore.
             }
@@ -103,7 +113,7 @@ class CordaRPCClient private constructor(
     private val rpcClient = RPCClient<CordaRPCOps>(
             tcpTransport(ConnectionDirection.Outbound(), hostAndPort, config = sslConfiguration),
             configuration.toRpcClientConfiguration(),
-            KRYO_RPC_CLIENT_CONTEXT
+            if (classLoader != null) KRYO_RPC_CLIENT_CONTEXT.withClassLoader(classLoader) else KRYO_RPC_CLIENT_CONTEXT
     )
 
     /**

client/rpc/src/main/kotlin/net/corda/client/rpc/internal/CordaRPCClientUtils.kt

@@ -10,4 +10,11 @@ fun createCordaRPCClientWithSsl(
         hostAndPort: NetworkHostAndPort,
         configuration: CordaRPCClientConfiguration = CordaRPCClientConfiguration.DEFAULT,
         sslConfiguration: SSLConfiguration? = null
-) = CordaRPCClient.createWithSsl(hostAndPort, configuration, sslConfiguration)
+) = CordaRPCClient.createWithSsl(hostAndPort, configuration, sslConfiguration)
+
+fun createCordaRPCClientWithSslAndClassLoader(
+        hostAndPort: NetworkHostAndPort,
+        configuration: CordaRPCClientConfiguration = CordaRPCClientConfiguration.DEFAULT,
+        sslConfiguration: SSLConfiguration? = null,
+        classLoader: ClassLoader? = null
+) = CordaRPCClient.createWithSslAndClassLoader(hostAndPort, configuration, sslConfiguration, classLoader)

client/rpc/src/main/kotlin/net/corda/client/rpc/internal/KryoClientSerializationScheme.kt

@@ -33,18 +33,19 @@ class KryoClientSerializationScheme : AbstractKryoSerializationScheme() {
 
     companion object {
         /** Call from main only. */
-        fun initialiseSerialization() {
-            nodeSerializationEnv = createSerializationEnv()
+        fun initialiseSerialization(classLoader: ClassLoader? = null) {
+            nodeSerializationEnv = createSerializationEnv(classLoader)
         }
 
-        fun createSerializationEnv(): SerializationEnvironment {
+        fun createSerializationEnv(classLoader: ClassLoader? = null): SerializationEnvironment {
             return SerializationEnvironmentImpl(
                     SerializationFactoryImpl().apply {
                         registerScheme(KryoClientSerializationScheme())
                         registerScheme(AMQPClientSerializationScheme(emptyList()))
                     },
-                    AMQP_P2P_CONTEXT,
-                    rpcClientContext = KRYO_RPC_CLIENT_CONTEXT)
+                    if (classLoader != null) AMQP_P2P_CONTEXT.withClassLoader(classLoader) else AMQP_P2P_CONTEXT,
+                    rpcClientContext = if (classLoader != null) KRYO_RPC_CLIENT_CONTEXT.withClassLoader(classLoader) else KRYO_RPC_CLIENT_CONTEXT)
+
         }
     }
 }

docs/source/shell.rst

@@ -9,7 +9,7 @@ Shell
 
 .. contents::
 
-The Corda shell is an embedded command line that allows an administrator to control and monitor a node. It is based on
+The Corda shell is an embedded or standalone command line that allows an administrator to control and monitor a node. It is based on
 the `CRaSH`_ shell and supports many of the same features. These features include:
 
 * Invoking any of the node's RPC methods
@@ -19,11 +19,22 @@ the `CRaSH`_ shell and supports many of the same features. These features includ
 * Viewing JMX metrics and monitoring exports
 * UNIX style pipes for both text and objects, an ``egrep`` command and a command for working with columnular data
 
+Permissions
+-----------
+
+When accessing the shell (embedded, standalone, via SSH) RPC permissions are required. This is because the shell actually communicates
+with the node using RPC calls.
+
+* Watching flows (``flow watch``) requires ``InvokeRpc.stateMachinesFeed``
+* Starting flows requires ``InvokeRpc.startTrackedFlowDynamic``, ``InvokeRpc.registeredFlows`` and ``InvokeRpc.wellKnownPartyFromX500Name``, as well as a
+  permission for the flow being started
+
 The shell via the local terminal
 --------------------------------
 
-In development mode, the shell will display in the node's terminal window. It may be disabled by passing the
-``--no-local-shell`` flag when running the node.
+In development mode, the shell will display in the node's terminal window.
+The shell connects to the node as 'shell' user with password 'shell' which is only available in dev mode.
+It may be disabled by passing the ``--no-local-shell`` flag when running the node.
 
 The shell via SSH
 -----------------
@@ -42,8 +53,8 @@ By default, the SSH server is *disabled*. To enable it, a port must be configure
 
 Authentication
 **************
-Users log in to shell via SSH using the same credentials as for RPC. This is because the shell actually communicates
-with the node using RPC calls. No RPC permissions are required to allow the connection and log in.
+Users log in to shell via SSH using the same credentials as for RPC.
+No RPC permissions are required to allow the connection and log in.
 
 The host key is loaded from the ``<node root directory>/sshkey/hostkey.pem`` file. If this file does not exist, it is
 generated automatically. In development mode, the seed may be specified to give the same results on the same computer
@@ -69,7 +80,7 @@ Where:
 
 The RPC password will be requested after a connection is established.
 
-:note: In development mode, restarting a node frequently may cause the host key to be regenerated. SSH usually saves
+.. note:: In development mode, restarting a node frequently may cause the host key to be regenerated. SSH usually saves
     trusted hosts and will refuse to connect in case of a change. This check can be disabled using the
     ``-o StrictHostKeyChecking=no`` flag. This option should never be used in production environment!
 
@@ -78,14 +89,99 @@ Windows
 
 Windows does not provide a built-in SSH tool. An alternative such as PuTTY should be used.
 
-Permissions
-***********
+The standalone shell
+------------------------------
+The standalone shell is a standalone application interacting with a Corda node via RPC calls.
+RPC node permissions are necessary for authentication and authorisation.
+Certain operations, such as starting flows, require access to CordApps jars.
 
-When accessing the shell via SSH, some additional RPC permissions are required:
+Starting the standalone shell
+*************************
+
+Run the following command from the terminal:
+
+Linux and MacOS
+^^^^^^^^^^^^^^^
+
+.. code:: bash
+
+    ./shell [--config-file PATH | --cordpass-directory PATH --commands-directory PATH --host HOST --port PORT
+             --user USER --password PASSWORD --sshd-port PORT --sshd-hostkey-directory PATH --keystore-password PASSWORD
+             --keystore-file FILE --truststore-password PASSWORD --truststore-file FILE | --help]
+
+Windows
+^^^^^^^
+
+.. code:: bash
+
+    shell.bat [--config-file PATH | --cordpass-directory PATH --commands-directory PATH --host HOST --port PORT
+             --user USER --password PASSWORD --sshd-port PORT --sshd-hostkey-directory PATH --keystore-password PASSWORD
+             --keystore-file FILE --truststore-password PASSWORD --truststore-file FILE | --help]
+
+Where:
+
+* ``config-file`` is the path to config file, used instead of providing the rest of command line options
+* ``cordpass-directory`` is the directory containing Cordapps jars, Cordapps are require when starting flows
+* ``commands-directory`` is the directory with additional CrAsH shell commands
+* ``host`` is the Corda node's host
+* ``port`` is the Corda node's port, specified in the ``node.conf`` file
+* ``user`` is the RPC username, if not provided it will be requested at startup
+* ``password`` is the RPC user password, if not provided it will be requested at startup
+* ``sshd-port`` instructs the standalone shell app to start SSH server on the given port, optional
+* ``sshd-hostkey-directory`` is the directory containing hostkey.pem file for SSH server
+* ``keystore-password`` the password to unlock the KeyStore file containing the standalone shell certificate and private key, optional, unencrypted RPC connection without SSL will be used if the option is not provided
+* ``keystore-file`` is the path to the KeyStore file
+* ``truststore-password`` the password to unlock the TrustStore file containing the Corda node certificate, optional, unencrypted RPC connection without SSL will be used if the option is not provided
+* ``truststore-file`` is the path to the TrustStore file
+* ``help`` prints Shell help
+
+The format of ``config-file``:
+
+.. code:: bash
+
+    node {
+        addresses {
+            rpc {
+                host : "localhost"
+                port : 10006
+            }
+        }
+    }
+    shell {
+	    workDir : /path/to/dir
+    }
+    extensions {
+        cordapps {
+            path : /path/to/cordapps/dir
+        }
+        sshd {
+            enabled : "false"
+            port : 2223
+        }
+    }
+    ssl {
+        keystore {
+            path: "/path/to/keystore"
+            type: "JKS"
+            password: password
+        }
+        trustore {
+            path: "/path/to/trusttore"
+            type: "JKS"
+            password: password
+        }
+    }
+    user : demo
+    password : demo
+
+
+Standalone Shell via SSH
+------------------------------------------
+The standalone shell can embed an SSH server which redirects interactions via RPC calls to the Corda node.
+To run SSH server use ``--sshd-port`` option when starting standalone shell or ``extensions.sshd`` entry in the configuration file.
+For connection to SSH refer to `Connecting to the shell`_.
+Certain operations (like starting Flows) will require Shell's ``--cordpass-directory`` to be configured correctly (see `Starting the standalone shell`_).
 
-* Watching flows (``flow watch``) requires ``InvokeRpc.stateMachinesFeed``
-* Starting flows requires ``InvokeRpc.startTrackedFlowDynamic`` and ``InvokeRpc.registeredFlows``, as well as a
-  permission for the flow being started
 
 Interacting with the node via the shell
 ---------------------------------------

node/build.gradle

@@ -65,6 +65,7 @@ dependencies {
     compile project(':node-api')
     compile project(":confidential-identities")
     compile project(':client:rpc')
+    compile project(':tools:shell')
     compile "net.corda.plugins:cordform-common:$gradle_plugins_version"
 
     // Log4J: logging framework (with SLF4J bindings)
@@ -102,10 +103,6 @@ dependencies {
         exclude group: "asm"
     }
 
-    // Jackson support: serialisation to/from JSON, YAML, etc
-    compile project(':client:jackson')
-    compile group: 'org.json', name: 'json', version: json_version
-
     // Coda Hale's Metrics: for monitoring of key statistics
     compile "io.dropwizard.metrics:metrics-core:3.1.2"
 
@@ -150,17 +147,6 @@ dependencies {
     // Netty: All of it.
     compile "io.netty:netty-all:$netty_version"
 
-    // CRaSH: An embeddable monitoring and admin shell with support for adding new commands written in Groovy.
-    compile("com.github.corda.crash:crash.shell:$crash_version") {
-        exclude group: "org.slf4j", module: "slf4j-jdk14"
-        exclude group: "org.bouncycastle"
-    }
-
-    compile("com.github.corda.crash:crash.connectors.ssh:$crash_version") {
-        exclude group: "org.slf4j", module: "slf4j-jdk14"
-        exclude group: "org.bouncycastle"
-    }
-
     // OkHTTP: Simple HTTP library.
     compile "com.squareup.okhttp3:okhttp:$okhttp_version"
 
@@ -175,9 +161,6 @@ dependencies {
     integrationTestCompile "junit:junit:$junit_version"
     integrationTestCompile "org.assertj:assertj-core:${assertj_version}"
 
-    // Jsh: Testing SSH server
-    integrationTestCompile group: 'com.jcraft', name: 'jsch', version: '0.1.54'
-
     // AgentLoader: dynamic loading of JVM agents
     compile group: 'com.ea.agentloader', name: 'ea-agent-loader', version: "${eaagentloader_version}"
 

node/src/integration-test/kotlin/net/corda/node/services/rpc/RpcSslTest.kt

@@ -5,8 +5,8 @@ import net.corda.client.rpc.internal.createCordaRPCClientWithSsl
 import net.corda.core.identity.CordaX500Name
 import net.corda.core.utilities.getOrThrow
 import net.corda.node.services.Permissions.Companion.all
-import net.corda.node.testsupport.withCertificates
-import net.corda.node.testsupport.withKeyStores
+import net.corda.testing.common.internal.withCertificates
+import net.corda.testing.common.internal.withKeyStores
 import net.corda.testing.driver.DriverParameters
 import net.corda.testing.driver.PortAllocation
 import net.corda.testing.driver.driver

node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt

@@ -42,6 +42,7 @@ import net.corda.node.services.FinalityHandler
 import net.corda.node.services.NotaryChangeHandler
 import net.corda.node.services.api.*
 import net.corda.node.services.config.*
+import net.corda.node.services.config.shell.toShellConfig
 import net.corda.node.services.events.NodeSchedulerService
 import net.corda.node.services.events.ScheduledActivityObserver
 import net.corda.node.services.identity.PersistentIdentityService
@@ -56,7 +57,6 @@ import net.corda.node.services.transactions.*
 import net.corda.node.services.upgrade.ContractUpgradeServiceImpl
 import net.corda.node.services.vault.NodeVaultService
 import net.corda.node.services.vault.VaultSoftLockManager
-import net.corda.node.shell.InteractiveShell
 import net.corda.node.utilities.AffinityExecutor
 import net.corda.node.utilities.JVMAgentRegistry
 import net.corda.node.utilities.NodeBuildProperties
@@ -67,6 +67,7 @@ import net.corda.nodeapi.internal.persistence.CordaPersistence
 import net.corda.nodeapi.internal.persistence.DatabaseConfig
 import net.corda.nodeapi.internal.persistence.HibernateConfiguration
 import net.corda.nodeapi.internal.storeLegalIdentity
+import net.corda.tools.shell.InteractiveShell
 import org.apache.activemq.artemis.utils.ReusableLatch
 import org.hibernate.type.descriptor.java.JavaTypeDescriptorRegistry
 import org.slf4j.Logger
@@ -258,7 +259,7 @@ abstract class AbstractNode(val configuration: NodeConfiguration,
             tokenizableServices = nodeServices + cordaServices + schedulerService
             registerCordappFlows(smm)
             _services.rpcFlows += cordappLoader.cordapps.flatMap { it.rpcFlows }
-            startShell(rpcOps)
+            startShell()
             Pair(StartedNodeImpl(this@AbstractNode, _services, nodeInfo, checkpointStorage, smm, attachments, network, database, rpcOps, flowStarter, notaryService), schedulerService)
         }
         networkMapUpdater = NetworkMapUpdater(services.networkMapCache,
@@ -296,9 +297,12 @@ abstract class AbstractNode(val configuration: NodeConfiguration,
      */
     protected abstract fun getRxIoScheduler(): Scheduler
 
-    open fun startShell(rpcOps: CordaRPCOps) {
+    open fun startShell() {
         if (configuration.shouldInitCrashShell()) {
-            InteractiveShell.startShell(configuration, rpcOps, securityManager, _services.identityService, _services.database)
+            if (configuration.rpcOptions.address == null) {
+                throw ConfigurationException("Cannot init CrashShell because node RPC address is not set (via 'rpcSettings' option).")
+            }
+            InteractiveShell.startShell(configuration.toShellConfig())
         }
     }