pytest: Add a new RPC interface to talk to grpc

This allows us to re-use existing tests (assuming the call and fields
are covered by `cln-rpc` and `cln-grpc`) to test the full roundtrip
from test over the grpc interface to the json-rpc interface and back
again.

You can switch to the grpc interface by setting the `CLN_TEST_GRPC`
environment variable to 1, but for now only very few shims are
implemented (due to the non-generated nature of LightningRpc).

Author: cdecker

contrib/pyln-testing/pyln/testing/grpc.py

@@ -0,0 +1,84 @@
+"""A drop-in replacement for the JSON-RPC LightningRpc
+"""
+
+from pyln.testing import node_pb2_grpc as pbgrpc
+from pyln.testing import node_pb2 as pb
+import grpc
+import json
+from google.protobuf.json_format import MessageToJson
+from pyln.testing import grpc2py
+
+
+DUMMY_CA_PEM = b"""-----BEGIN CERTIFICATE-----
+MIIBcTCCARigAwIBAgIJAJhah1bqO05cMAoGCCqGSM49BAMCMBYxFDASBgNVBAMM
+C2NsbiBSb290IENBMCAXDTc1MDEwMTAwMDAwMFoYDzQwOTYwMTAxMDAwMDAwWjAW
+MRQwEgYDVQQDDAtjbG4gUm9vdCBDQTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA
+BPF4JrGsOsksgsYM1NNdUdLESwOxkzyD75Rnj/g7sFEVYXewcmyB3MRGCBx2a3/7
+ft2Xu2ED6WigajaHlnSvfUyjTTBLMBkGA1UdEQQSMBCCA2NsboIJbG9jYWxob3N0
+MB0GA1UdDgQWBBRcTjvqVodamGirO6sX1rOR02LwXzAPBgNVHRMBAf8EBTADAQH/
+MAoGCCqGSM49BAMCA0cAMEQCICDvV5iFw/nmJdl6rlEEGAdBdZqjxD0tV6U/FvuL
+7PycAiASEMtsFtpfiUvxveBkOGt7AN32GP/Z75l+GhYXh7L1ig==
+-----END CERTIFICATE-----"""
+
+
+DUMMY_CA_KEY_PEM = b"""-----BEGIN PRIVATE KEY-----
+MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgqbU7LQsRcvmI5vE5
+MBBNK3imhIU2jmAczgvLuBi/Ys+hRANCAATxeCaxrDrJLILGDNTTXVHSxEsDsZM8
+g++UZ4/4O7BRFWF3sHJsgdzERggcdmt/+37dl7thA+looGo2h5Z0r31M
+-----END PRIVATE KEY-----"""
+
+
+DUMMY_CLIENT_KEY_PEM = b"""-----BEGIN PRIVATE KEY-----
+MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgIEdQyKso8PaD1kiz
+xxFEcKiTvTg+bej4Nc/GqnXipcGhRANCAARGoUNSnWx1qgt4RiVG8tOMX1vpKvhr
+OLcUJ92T++kIFZchZvcTXwnlNiTAQg3ukL+RYyG5Q1PaYrYRVlOtl1T0
+-----END PRIVATE KEY-----"""
+
+
+DUMMY_CLIENT_PEM = b"""-----BEGIN CERTIFICATE-----
+MIIBRDCB7KADAgECAgkA8SsXq7IZfi8wCgYIKoZIzj0EAwIwFjEUMBIGA1UEAwwL
+Y2xuIFJvb3QgQ0EwIBcNNzUwMTAxMDAwMDAwWhgPNDA5NjAxMDEwMDAwMDBaMBox
+GDAWBgNVBAMMD2NsbiBncnBjIFNlcnZlcjBZMBMGByqGSM49AgEGCCqGSM49AwEH
+A0IABEahQ1KdbHWqC3hGJUby04xfW+kq+Gs4txQn3ZP76QgVlyFm9xNfCeU2JMBC
+De6Qv5FjIblDU9pithFWU62XVPSjHTAbMBkGA1UdEQQSMBCCA2NsboIJbG9jYWxo
+b3N0MAoGCCqGSM49BAMCA0cAMEQCICTU/YAs35cb6DRdZNzO1YbEt77uEjcqMRca
+Hh6kK99RAiAKOQOkGnoAICjBmBJeC/iC4/+hhhkWZtFgbC3Jg5JD0w==
+-----END CERTIFICATE-----"""
+
+
+class LightningGrpc(object):
+    def __init__(
+        self,
+        host: str,
+        port: int,
+        root_certificates: bytes = DUMMY_CA_PEM,
+        private_key: bytes = DUMMY_CLIENT_KEY_PEM,
+        certificate_chain: bytes = DUMMY_CLIENT_PEM,
+    ):
+        self.credentials = grpc.ssl_channel_credentials(
+            root_certificates=root_certificates,
+            private_key=private_key,
+            certificate_chain=certificate_chain,
+        )
+        self.channel = grpc.secure_channel(
+            f"{host}:{port}",
+            self.credentials,
+            options=(("grpc.ssl_target_name_override", "cln"),),
+        )
+        self.stub = pbgrpc.NodeStub(self.channel)
+
+    def getinfo(self):
+        return grpc2py.getinfo2py(
+            self.stub.Getinfo(pb.GetinfoRequest())
+        )
+
+    def connect(self, peer_id, host=None, port=None):
+        """
+        Connect to {peer_id} at {host} and {port}.
+        """
+        payload = pb.ConnectRequest(
+            id=peer_id,
+            host=host,
+            port=port
+        )
+        return grpc2py.connect2py(self.stub.ConnectPeer(payload))

contrib/pyln-testing/pyln/testing/utils.py

@@ -10,6 +10,7 @@
 from decimal import Decimal
 from pyln.client import LightningRpc
 from pyln.client import Millisatoshi
+from pyln.testing import grpc
 
 import ephemeral_port_reserve  # type: ignore
 import json
@@ -538,7 +539,15 @@ def getnewaddress(self):
 
 
 class LightningD(TailableProc):
-    def __init__(self, lightning_dir, bitcoindproxy, port=9735, random_hsm=False, node_id=0):
+    def __init__(
+            self,
+            lightning_dir,
+            bitcoindproxy,
+            port=9735,
+            random_hsm=False,
+            node_id=0,
+            grpc_port=None
+    ):
         # We handle our own version of verbose, below.
         TailableProc.__init__(self, lightning_dir, verbose=False)
         self.executable = 'lightningd'
@@ -564,6 +573,9 @@ def __init__(self, lightning_dir, bitcoindproxy, port=9735, random_hsm=False, no
             'bitcoin-datadir': lightning_dir,
         }
 
+        if grpc_port is not None:
+            opts['grpc-port'] = grpc_port
+
         for k, v in opts.items():
             self.opts[k] = v
 
@@ -693,19 +705,22 @@ def __init__(self, node_id, lightning_dir, bitcoind, executor, valgrind, may_fai
         self.allow_bad_gossip = allow_bad_gossip
         self.allow_warning = allow_warning
         self.db = db
+        self.lightning_dir = Path(lightning_dir)
 
         # Assume successful exit
         self.rc = 0
 
-        socket_path = os.path.join(lightning_dir, TEST_NETWORK, "lightning-rpc").format(node_id)
-        self.rpc = PrettyPrintingLightningRpc(socket_path, self.executor, jsonschemas=jsonschemas)
+        # Ensure we have an RPC we can use to talk to the node
+        self._create_rpc(jsonschemas)
 
         self.gossip_store = GossipStore(Path(lightning_dir, TEST_NETWORK, "gossip_store"))
 
         self.daemon = LightningD(
             lightning_dir, bitcoindproxy=bitcoind.get_proxy(),
-            port=port, random_hsm=random_hsm, node_id=node_id
+            port=port, random_hsm=random_hsm, node_id=node_id,
+            grpc_port=self.grpc_port,
         )
+
         # If we have a disconnect string, dump it to a file for daemon.
         if disconnect:
             self.daemon.disconnect_file = os.path.join(lightning_dir, TEST_NETWORK, "dev_disconnect")
@@ -751,6 +766,47 @@ def __init__(self, node_id, lightning_dir, bitcoind, executor, valgrind, may_fai
             if SLOW_MACHINE:
                 self.daemon.cmd_prefix += ['--read-inline-info=no']
 
+    def _create_rpc(self, jsonschemas):
+        """Prepares anything related to the RPC.
+        """
+        if os.environ.get('CLN_TEST_GRPC') == '1':
+            logging.info("Switching to GRPC based RPC for tests")
+            self._create_grpc_rpc()
+        else:
+            self._create_jsonrpc_rpc(jsonschemas)
+
+    def _create_grpc_rpc(self):
+        self.grpc_port = reserve_unused_port()
+        d = self.lightning_dir / TEST_NETWORK
+        d.mkdir(parents=True, exist_ok=True)
+
+        # Copy all the certificates and keys into place:
+        with (d / "ca.pem").open(mode='wb') as f:
+            f.write(grpc.DUMMY_CA_PEM)
+
+        with (d / "ca-key.pem").open(mode='wb') as f:
+            f.write(grpc.DUMMY_CA_KEY_PEM)
+
+        # Now the node will actually start up and use them, so we can
+        # create the RPC instance.
+        self.rpc = grpc.LightningGrpc(
+            host='localhost',
+            port=self.grpc_port,
+            root_certificates=grpc.DUMMY_CA_PEM,
+            private_key=grpc.DUMMY_CLIENT_KEY_PEM,
+            certificate_chain=grpc.DUMMY_CLIENT_PEM
+        )
+
+    def _create_jsonrpc_rpc(self, jsonschemas):
+        socket_path = self.lightning_dir / TEST_NETWORK / "lightning-rpc"
+        self.grpc_port = None
+
+        self.rpc = PrettyPrintingLightningRpc(
+            str(socket_path),
+            self.executor,
+            jsonschemas=jsonschemas
+        )
+
     def connect(self, remote_node):
         self.rpc.connect(remote_node.info['id'], '127.0.0.1', remote_node.daemon.port)
 

tools/check-spelling.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-if git --no-pager grep -nHiE 'l[ightn]{6}g|l[ightn]{8}g|ilghtning|lgihtning|lihgtning|ligthning|lighnting|lightinng|lightnnig|lightnign' -- . ':!tools/check-spelling.sh' | grep -vE "highlighting"; then
+if git --no-pager grep -nHiE 'l[ightn]{6}g|l[ightn]{8}g|ilghtning|lgihtning|lihgtning|ligthning|lighnting|lightinng|lightnnig|lightnign' -- . ':!tools/check-spelling.sh' | grep -vE "highlighting|LightningGrpc"; then
     echo "Identified a likely misspelling of the word \"lightning\" (see above). Please fix."
     echo "Is this warning incorrect? Please teach tools/check-spelling.sh about the exciting new word."
     exit 1