From d01b2c21a7f5b0190eece0628edee40e8929b678 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Fri, 14 Jan 2022 19:45:30 +0100 Subject: [PATCH] cln-grpc: Add generation of grpc protobuf file from schema --- Makefile | 1 + cln-grpc/Makefile | 11 ++ cln-grpc/proto/node.proto | 184 ++++++++++++++++++++++++++++++ cln-grpc/proto/primitives.proto | 33 ++++++ contrib/msggen/msggen/__main__.py | 8 ++ contrib/msggen/msggen/grpc.py | 153 +++++++++++++++++++++++++ requirements.lock | 71 +++++++----- 7 files changed, 431 insertions(+), 30 deletions(-) create mode 100644 cln-grpc/Makefile create mode 100644 cln-grpc/proto/node.proto create mode 100644 cln-grpc/proto/primitives.proto create mode 100644 contrib/msggen/msggen/grpc.py diff --git a/Makefile b/Makefile index 35c5b34ae245..62fec37c111d 100644 --- a/Makefile +++ b/Makefile @@ -356,6 +356,7 @@ ifneq ($(FUZZING),0) endif ifneq ($(RUST),0) include cln-rpc/Makefile + include cln-grpc/Makefile endif # We make pretty much everything depend on these. diff --git a/cln-grpc/Makefile b/cln-grpc/Makefile new file mode 100644 index 000000000000..9eb6bc26e4de --- /dev/null +++ b/cln-grpc/Makefile @@ -0,0 +1,11 @@ +cln-grpc-wrongdir: + $(MAKE) -C .. cln-grpc-all + +CLN_GRPC_EXAMPLES := +CLN_GRPC_GENALL = cln-grpc/proto/node.proto +DEFAULT_TARGETS += $(CLN_GRPC_EXAMPLES) $(CLN_GRPC_GENALL) + +$(CLN_GRPC_GENALL): $(JSON_SCHEMA) + PYTHONPATH=contrib/msggen python3 contrib/msggen/msggen/__main__.py + +cln-grpc-all: ${CLN_GRPC_GENALL} ${CLN_GRPC_EXAMPLES} diff --git a/cln-grpc/proto/node.proto b/cln-grpc/proto/node.proto new file mode 100644 index 000000000000..e7d7f5267565 --- /dev/null +++ b/cln-grpc/proto/node.proto @@ -0,0 +1,184 @@ +syntax = "proto3"; +package cln; + +// This file was automatically derived from the JSON-RPC schemas in +// `doc/schemas`. Do not edit this file manually as it would get +// overwritten. + +import "primitives.proto"; + +service Node { + rpc Getinfo(GetinfoRequest) returns (GetinfoResponse) {} + rpc ListFunds(ListfundsRequest) returns (ListfundsResponse) {} + rpc ListChannels(ListchannelsRequest) returns (ListchannelsResponse) {} + rpc AddGossip(AddgossipRequest) returns (AddgossipResponse) {} + rpc AutoCleanInvoice(AutocleaninvoiceRequest) returns (AutocleaninvoiceResponse) {} + rpc CheckMessage(CheckmessageRequest) returns (CheckmessageResponse) {} + rpc Close(CloseRequest) returns (CloseResponse) {} +} + +message GetinfoRequest { +} + +message GetinfoResponse { + bytes id = 1; + string alias = 2; + bytes color = 3; + uint32 num_peers = 4; + uint32 num_pending_channels = 5; + uint32 num_active_channels = 6; + uint32 num_inactive_channels = 7; + string version = 8; + string lightning_dir = 9; + uint32 blockheight = 10; + string network = 11; + Amount fees_collected_msat = 12; + repeated GetinfoAddress address = 13; + repeated GetinfoBinding binding = 14; + optional string warning_bitcoind_sync = 15; + optional string warning_lightningd_sync = 16; +} + +message GetinfoAddress { + // Getinfo.address[].type + enum GetinfoAddressType { + DNS = 0; + IPV4 = 1; + IPV6 = 2; + TORV2 = 3; + TORV3 = 4; + WEBSOCKET = 5; + } + GetinfoAddressType item_type = 1; + uint32 port = 2; + optional string address = 3; +} + +message GetinfoBinding { + // Getinfo.binding[].type + enum GetinfoBindingType { + LOCAL_SOCKET = 0; + IPV4 = 1; + IPV6 = 2; + TORV2 = 3; + TORV3 = 4; + } + GetinfoBindingType item_type = 1; + optional string address = 2; + optional uint32 port = 3; + optional string socket = 4; +} + +message ListfundsRequest { + optional bool spent = 1; +} + +message ListfundsResponse { + repeated ListfundsOutputs outputs = 1; + repeated ListfundsChannels channels = 2; +} + +message ListfundsOutputs { + // ListFunds.outputs[].status + enum ListfundsOutputsStatus { + UNCONFIRMED = 0; + CONFIRMED = 1; + SPENT = 2; + } + bytes txid = 1; + uint32 output = 2; + Amount amount_msat = 3; + bytes scriptpubkey = 4; + optional string address = 5; + optional bytes redeemscript = 6; + ListfundsOutputsStatus status = 7; + optional uint32 blockheight = 8; +} + +message ListfundsChannels { + bytes peer_id = 1; + Amount our_amount_msat = 2; + Amount amount_msat = 3; + bytes funding_txid = 4; + uint32 funding_output = 5; + bool connected = 6; + ChannelState state = 7; + optional string short_channel_id = 8; +} + +message ListchannelsRequest { + optional string short_channel_id = 1; + optional bytes source = 2; + optional bytes destination = 3; +} + +message ListchannelsResponse { + repeated ListchannelsChannels channels = 1; +} + +message ListchannelsChannels { + bytes source = 1; + bytes destination = 2; + bool public = 3; + Amount amount_msat = 4; + uint32 message_flags = 5; + uint32 channel_flags = 6; + bool active = 7; + uint32 last_update = 8; + uint32 base_fee_millisatoshi = 9; + uint32 fee_per_millionth = 10; + uint32 delay = 11; + Amount htlc_minimum_msat = 12; + optional Amount htlc_maximum_msat = 13; + bytes features = 14; +} + +message AddgossipRequest { + bytes message = 1; +} + +message AddgossipResponse { +} + +message AutocleaninvoiceRequest { + optional uint64 expired_by = 1; + optional uint64 cycle_seconds = 2; +} + +message AutocleaninvoiceResponse { + bool enabled = 1; + optional uint64 expired_by = 2; + optional uint64 cycle_seconds = 3; +} + +message CheckmessageRequest { + string message = 1; + string zbase = 2; + optional bytes pubkey = 3; +} + +message CheckmessageResponse { + bool verified = 1; + optional bytes pubkey = 2; +} + +message CloseRequest { + bytes id = 1; + optional uint32 unilateraltimeout = 2; + optional string destination = 3; + optional string fee_negotiation_step = 4; + optional bytes wrong_funding = 5; + optional bool force_lease_closed = 6; +} + +message CloseResponse { + // Close.type + enum CloseType { + MUTUAL = 0; + UNILATERAL = 1; + UNOPENED = 2; + } + CloseType item_type = 1; + optional bytes tx = 2; + optional bytes txid = 3; +} diff --git a/cln-grpc/proto/primitives.proto b/cln-grpc/proto/primitives.proto new file mode 100644 index 000000000000..d08e10bf76b0 --- /dev/null +++ b/cln-grpc/proto/primitives.proto @@ -0,0 +1,33 @@ +syntax = "proto3"; +package cln; + +message Amount { + oneof unit { + uint64 millisatoshi = 1; + uint64 satoshi = 2; + uint64 bitcoin = 3; + bool all = 4; + bool any = 5; + } +} + +enum ChannelSide { + IN = 0; + OUT = 1; +} + +enum ChannelState { + Openingd = 0; + ChanneldAwaitingLockin = 1; + ChanneldNormal = 2; + ChanneldShuttingDown = 3; + ClosingdSigexchange = 4; + ClosingdComplete = 5; + AwaitingUnilateral = 6; + FundingSpendSeen = 7; + Onchain = 8; + DualopendOpenInit = 9; + DualopendAwaitingLockin = 10; +} + +message ChannelStateChangeCause {} \ No newline at end of file diff --git a/contrib/msggen/msggen/__main__.py b/contrib/msggen/msggen/__main__.py index 982607aba820..219ddf4a5a07 100644 --- a/contrib/msggen/msggen/__main__.py +++ b/contrib/msggen/msggen/__main__.py @@ -1,4 +1,5 @@ from msggen.model import Method, CompositeField, Service +from msggen.grpc import GrpcGenerator from msggen.rust import RustGenerator from pathlib import Path import subprocess @@ -125,6 +126,12 @@ def load_jsonrpc_service(): return service +def gengrpc(service): + """Load all mapped RPC methods, wrap them in a Service, and split them into messages. + """ + fname = repo_root() / "cln-grpc" / "proto" / "node.proto" + dest = open(fname, "w") + GrpcGenerator(dest).generate(service) def genrustjsonrpc(service): fname = repo_root() / "cln-rpc" / "src" / "model.rs" dest = open(fname, "w") @@ -133,6 +140,7 @@ def genrustjsonrpc(service): def run(): service = load_jsonrpc_service() + gengrpc(service) genrustjsonrpc(service) diff --git a/contrib/msggen/msggen/grpc.py b/contrib/msggen/msggen/grpc.py new file mode 100644 index 000000000000..d90b14f4d318 --- /dev/null +++ b/contrib/msggen/msggen/grpc.py @@ -0,0 +1,153 @@ +# A grpc model +from .model import ArrayField, Field, CompositeField, EnumField, PrimitiveField, Service +from typing import TextIO, List +from textwrap import indent, dedent +import re +import logging + + +typemap = { + 'boolean': 'bool', + 'hex': 'bytes', + 'msat': 'Amount', + 'number': 'i64', + 'pubkey': 'bytes', + 'short_channel_id': 'string', + 'signature': 'bytes', + 'string': 'string', + 'txid': 'bytes', + 'u8': 'uint32', # Yep, this is the smallest integer type in grpc... + 'u32': 'uint32', + 'u64': 'uint64', + 'u16': 'uint32', # Yeah, I know... +} + + +# Manual overrides for some of the auto-generated types for paths +overrides = { + 'ListPeers.peers[].channels[].state_changes[].old_state': "ChannelState", + 'ListPeers.peers[].channels[].state_changes[].new_state': "ChannelState", + 'ListPeers.peers[].channels[].state_changes[].cause': "ChannelStateChangeCause", + 'ListPeers.peers[].channels[].opener': "ChannelSide", + 'ListPeers.peers[].channels[].closer': "ChannelSide", + 'ListPeers.peers[].channels[].features[]': "string", + 'ListFunds.channels[].state': 'ChannelState', +} + + +class GrpcGenerator: + """A generator that generates protobuf files. + """ + + def __init__(self, dest: TextIO): + self.dest = dest + self.logger = logging.getLogger("msggen.grpc.GrpcGenerator") + + def write(self, text: str, cleanup: bool = True) -> None: + if cleanup: + self.dest.write(dedent(text)) + else: + self.dest.write(text) + + def gather_types(self, service): + """Gather all types that might need to be defined. + """ + + def gather_subfields(field: Field) -> List[Field]: + fields = [field] + + if isinstance(field, CompositeField): + for f in field.fields: + fields.extend(gather_subfields(f)) + elif isinstance(field, ArrayField): + fields = [] + fields.extend(gather_subfields(field.itemtype)) + + return fields + + types = [] + for method in service.methods: + types.extend([method.request, method.response]) + for field in method.request.fields: + types.extend(gather_subfields(field)) + for field in method.response.fields: + types.extend(gather_subfields(field)) + return types + + def generate_service(self, service: Service) -> None: + self.write(f""" + service {service.name} {{ + """) + + for method in service.methods: + self.write( + f" rpc {method.name}({method.request.typename}) returns ({method.response.typename}) {{}}\n", + cleanup=False, + ) + + self.write(f"""}} + """) + + def generate_enum(self, e: EnumField, indent=0): + self.logger.debug(f"Generating enum {e}") + prefix = "\t" * indent + self.write(f"{prefix}// {e.path}\n", False) + self.write(f"{prefix}enum {e.typename} {{\n", False) + + for i, v in enumerate(e.variants): + self.logger.debug(f"Generating enum variant {v}") + self.write(f"{prefix}\t{v.normalized()} = {i};\n", False) + + self.write(f"""{prefix}}}\n""", False) + + def generate_message(self, message: CompositeField): + self.write(f""" + message {message.typename} {{ + """) + + # Declare enums inline so they are scoped correctly in C++ + for i, f in enumerate(message.fields): + if isinstance(f, EnumField) and f.path not in overrides.keys(): + self.generate_enum(f, indent=1) + + for i, f in enumerate(message.fields): + opt = "optional " if not f.required else "" + if isinstance(f, ArrayField): + typename = typemap.get(f.itemtype.typename, f.itemtype.typename) + if f.path in overrides: + typename = overrides[f.path] + self.write(f"\trepeated {typename} {f.normalized()} = {i+1};\n", False) + elif isinstance(f, PrimitiveField): + typename = typemap.get(f.typename, f.typename) + if f.path in overrides: + typename = overrides[f.path] + self.write(f"\t{opt}{typename} {f.normalized()} = {i+1};\n", False) + elif isinstance(f, EnumField): + typename = f.typename + if f.path in overrides: + typename = overrides[f.path] + self.write(f"\t{opt}{typename} {f.normalized()} = {i+1};\n", False) + + self.write(f"""}} + """) + + def generate(self, service: Service) -> None: + """Generate the GRPC protobuf file and write to `dest` + """ + self.write(f"""syntax = "proto3";\npackage cln;\n""") + self.write(""" + // This file was automatically derived from the JSON-RPC schemas in + // `doc/schemas`. Do not edit this file manually as it would get + // overwritten. + + """) + + for i in service.includes: + self.write(f"import \"{i}\";\n") + + self.generate_service(service) + + fields = self.gather_types(service) + + for message in [f for f in fields if isinstance(f, CompositeField)]: + self.generate_message(message) diff --git a/requirements.lock b/requirements.lock index cc193eea8c3d..a6434caa4193 100644 --- a/requirements.lock +++ b/requirements.lock @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # -# pip-compile --output-file=requirements.lock requirements.txt +# pip-compile --output-file=requirements.lock requirements.in # alabaster==0.7.12 # via sphinx @@ -15,9 +15,9 @@ attrs==21.2.0 babel==2.9.1 # via sphinx base58==2.0.1 - # via pyln.proto + # via -r requirements.in bitstring==3.1.9 - # via pyln.proto + # via -r requirements.in certifi==2021.5.30 # via requests cffi==1.14.6 @@ -27,17 +27,17 @@ cffi==1.14.6 charset-normalizer==2.0.6 # via requests cheroot==8.5.2 - # via pyln-testing + # via -r requirements.in click==7.1.2 # via flask coincurve==13.0.0 - # via pyln.proto + # via -r requirements.in commonmark==0.9.1 # via recommonmark crc32c==2.2.post0 - # via -r requirements.txt + # via -r requirements.in cryptography==3.4.8 - # via pyln.proto + # via -r requirements.in docutils==0.17.1 # via # recommonmark @@ -45,15 +45,21 @@ docutils==0.17.1 entrypoints==0.3 # via flake8 ephemeral-port-reserve==1.1.1 - # via pyln-testing + # via -r requirements.in execnet==1.9.0 # via pytest-xdist flake8==3.7.9 - # via -r requirements.txt + # via -r requirements.in flaky==3.7.0 - # via pyln-testing + # via -r requirements.in flask==1.1.4 - # via pyln-testing + # via -r requirements.in +grpcio==1.34.0 + # via + # -r requirements.in + # grpcio-tools +grpcio-tools==1.34.0 + # via -r requirements.in idna==3.2 # via requests imagesize==1.2.0 @@ -70,9 +76,9 @@ jinja2==2.11.3 # mrkd # sphinx jsonschema==3.2.0 - # via pyln-testing + # via -r requirements.in mako==1.1.5 - # via -r requirements.txt + # via -r requirements.in markupsafe==2.0.1 # via # jinja2 @@ -88,9 +94,9 @@ more-itertools==8.10.0 # cheroot # jaraco.functools mrkd==0.1.6 - # via -r requirements.txt + # via -r requirements.in mypy==0.910 - # via pyln.proto + # via -r requirements.in mypy-extensions==0.4.3 # via mypy packaging==21.0 @@ -101,10 +107,12 @@ plac==1.3.3 # via mrkd pluggy==0.13.1 # via pytest +protobuf==3.19.3 + # via grpcio-tools psutil==5.7.3 - # via pyln-testing + # via -r requirements.in psycopg2-binary==2.8.6 - # via pyln-testing + # via -r requirements.in py==1.10.0 # via # pytest @@ -112,7 +120,9 @@ py==1.10.0 pycodestyle==2.5.0 # via flake8 pycparser==2.20 - # via cffi + # via + # -r requirements.in + # cffi pyflakes==2.1.1 # via flake8 pygments==2.10.0 @@ -124,10 +134,10 @@ pyparsing==2.4.7 pyrsistent==0.18.0 # via jsonschema pysocks==1.7.1 - # via pyln.proto + # via -r requirements.in pytest==6.1.2 # via - # pyln-testing + # -r requirements.in # pytest-forked # pytest-rerunfailures # pytest-timeout @@ -135,22 +145,23 @@ pytest==6.1.2 pytest-forked==1.3.0 # via pytest-xdist pytest-rerunfailures==9.1.1 - # via pyln-testing + # via -r requirements.in pytest-timeout==1.4.2 - # via pyln-testing + # via -r requirements.in pytest-xdist==2.2.1 - # via pyln-testing + # via -r requirements.in python-bitcoinlib==0.11.0 - # via pyln-testing + # via -r requirements.in pytz==2021.1 # via babel recommonmark==0.7.1 - # via pyln-client + # via -r requirements.in requests==2.26.0 # via sphinx six==1.16.0 # via # cheroot + # grpcio # jsonschema snowballstemmer==2.1.0 # via sphinx @@ -169,15 +180,15 @@ sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.5 # via sphinx toml==0.10.2 - # via pytest -typed-ast==1.4.3 - # via mypy + # via + # mypy + # pytest typing-extensions==3.10.0.2 # via mypy urllib3==1.26.7 # via requests websocket-client==1.2.1 - # via -r requirements.txt + # via -r requirements.in werkzeug==1.0.1 # via flask