bolt: update to BOLT spec changes (extract format + type specifications)

updates the bolt version to 6639cef095a2ecc7b8f0c48c6e7f2f906fbfbc58.

this requires us to use the new bolt parser at generate-bolt.py
and updates to all of the type specifications (ie. from u8 -> byte)

CHANGELOG.md

@@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- build: now requires `python3-mako` to be installed, i.e. `sudo apt-get install python3-mako`
+
 ### Deprecated
 
 Note: You should always set `allow-deprecated-apis=false` to test for

Makefile

@@ -15,7 +15,7 @@ CCANDIR := ccan
 
 # Where we keep the BOLT RFCs
 BOLTDIR := ../lightning-rfc/
-BOLTVERSION := 309e86d471faf90c1f6c531701c16887684badb9
+BOLTVERSION := 6639cef095a2ecc7b8f0c48c6e7f2f906fbfbc58
 
 -include config.vars
 
@@ -185,6 +185,8 @@ ALL_GEN_HEADERS += gen_version.h
 CDUMP_OBJS := ccan-cdump.o ccan-strmap.o
 
 WIRE_GEN := tools/generate-wire.py
+BOLT_GEN := tools/generate-bolts.py
+BOLT_DEPS := $(BOLT_GEN)
 
 ALL_PROGRAMS =
 
@@ -272,6 +274,13 @@ check-hdr-include-order/%: %
 check-makefile:
 	@if [ x"$(CCANDIR)/config.h `find $(CCANDIR)/ccan -name '*.h' | grep -v /test/ | LC_ALL=C sort | tr '\n' ' '`" != x"$(CCAN_HEADERS) " ]; then echo CCAN_HEADERS incorrect; exit 1; fi
 
+# Check if they've installed 'mako' dependency, useful for
+# keeping your build from clobbering itself
+check-bolt-dependency:
+	@python3 -c "import mako"
+
+BOLT_DEPS += check-bolt-dependency
+
 # Any mention of BOLT# must be followed by an exact quote, modulo whitespace.
 bolt-check/%: % bolt-precheck tools/check-bolt
 	@if [ -d .tmp.lightningrfc ]; then tools/check-bolt .tmp.lightningrfc $<; else echo "Not checking BOLTs: BOLTDIR $(BOLTDIR) does not exist" >&2; fi

README.md

@@ -59,7 +59,8 @@ For the impatient here's the gist of it for Ubuntu and Debian:
     sudo apt-get update
     sudo apt-get install -y \
       autoconf automake build-essential git libtool libgmp-dev \
-      libsqlite3-dev python python3 net-tools zlib1g-dev libsodium-dev
+      libsqlite3-dev python python3 net-tools zlib1g-dev libsodium-dev \
+      python3-mako
     git clone https://github.com/ElementsProject/lightning.git
     cd lightning
     ./configure

common/channel_config.h

@@ -9,28 +9,28 @@
  *
  * 1. type: 32 (`open_channel`)
  * 2. data:
- *    * [`32`:`chain_hash`]
- *    * [`32`:`temporary_channel_id`]
- *    * [`8`:`funding_satoshis`]
- *    * [`8`:`push_msat`]
- *    * [`8`:`dust_limit_satoshis`]
- *    * [`8`:`max_htlc_value_in_flight_msat`]
- *    * [`8`:`channel_reserve_satoshis`]
- *    * [`8`:`htlc_minimum_msat`]
- *    * [`4`:`feerate_per_kw`]
- *    * [`2`:`to_self_delay`]
- *    * [`2`:`max_accepted_htlcs`]
+ *    * [`chain_hash`:`chain_hash`]
+ *    * [`32*byte`:`temporary_channel_id`]
+ *    * [`u64`:`funding_satoshis`]
+ *    * [`u64`:`push_msat`]
+ *    * [`u64`:`dust_limit_satoshis`]
+ *    * [`u64`:`max_htlc_value_in_flight_msat`]
+ *    * [`u64`:`channel_reserve_satoshis`]
+ *    * [`u64`:`htlc_minimum_msat`]
+ *    * [`u32`:`feerate_per_kw`]
+ *    * [`u16`:`to_self_delay`]
+ *    * [`u16`:`max_accepted_htlcs`]
  *...
  * 1. type: 33 (`accept_channel`)
  * 2. data:
- *    * [`32`:`temporary_channel_id`]
- *    * [`8`:`dust_limit_satoshis`]
- *    * [`8`:`max_htlc_value_in_flight_msat`]
- *    * [`8`:`channel_reserve_satoshis`]
- *    * [`8`:`htlc_minimum_msat`]
- *    * [`4`:`minimum_depth`]
- *    * [`2`:`to_self_delay`]
- *    * [`2`:`max_accepted_htlcs`]
+ *    * [`32*byte`:`temporary_channel_id`]
+ *    * [`u64`:`dust_limit_satoshis`]
+ *    * [`u64`:`max_htlc_value_in_flight_msat`]
+ *    * [`u64`:`channel_reserve_satoshis`]
+ *    * [`u64`:`htlc_minimum_msat`]
+ *    * [`u32`:`minimum_depth`]
+ *    * [`u16`:`to_self_delay`]
+ *    * [`u16`:`max_accepted_htlcs`]
  */
 struct channel_config {
 	/* Database ID */

common/sphinx.c

@@ -464,11 +464,11 @@ u8 *create_onionreply(const tal_t *ctx, const struct secret *shared_secret,
 	 * the following fields:
 	 *
 	 * 1. data:
-	 *    * [`32`:`hmac`]
-	 *    * [`2`:`failure_len`]
-	 *    * [`failure_len`:`failuremsg`]
-	 *    * [`2`:`pad_len`]
-	 *    * [`pad_len`:`pad`]
+	 *    * [`32*byte`:`hmac`]
+	 *    * [`u16`:`failure_len`]
+	 *    * [`failure_len*byte`:`failuremsg`]
+	 *    * [`u16`:`pad_len`]
+	 *    * [`pad_len*byte`:`pad`]
 	 */
 	towire_u16(&payload, msglen);
 	towire(&payload, failure_msg, msglen);

common/sphinx.h

@@ -42,9 +42,9 @@ enum route_next_case {
  *
  * 1. type: `hops_data`
  * 2. data:
- *    * [`1`:`realm`]
- *    * [`32`:`per_hop`]
- *    * [`32`:`HMAC`]
+ *    * [`byte`:`realm`]
+ *    * [`32*byte`:`per_hop`]
+ *    * [`32*byte`:`HMAC`]
  *    * ...
  *    * `filler`
  *
@@ -59,10 +59,10 @@ enum route_next_case {
  *
  * 1. type: `per_hop` (for `realm` 0)
  * 2. data:
- *    * [`8`:`short_channel_id`]
- *    * [`8`:`amt_to_forward`]
- *    * [`4`:`outgoing_cltv_value`]
- *    * [`12`:`padding`]
+ *    * [`short_channel_id`:`short_channel_id`]
+ *    * [`u64`:`amt_to_forward`]
+ *    * [`u32`:`outgoing_cltv_value`]
+ *    * [`12*byte`:`padding`]
  */
 struct hop_data {
 	u8 realm;

devtools/Makefile

@@ -4,6 +4,8 @@ DEVTOOLS := devtools/bolt11-cli devtools/decodemsg devtools/onion devtools/dump-
 DEVTOOLS_TOOL_SRC := $(DEVTOOLS:=.c)
 DEVTOOLS_TOOL_OBJS := $(DEVTOOLS_TOOL_SRC:.c=.o)
 
+DEVTOOLS_BOLT_DEPS += $(BOLT_DEPS) tools/gen/print_impl_template tools/gen/print_header_template
+
 DEVTOOLS_COMMON_OBJS :=				\
 	common/amount.o				\
 	common/base32.o				\
@@ -24,17 +26,17 @@ DEVTOOLS_COMMON_OBJS :=				\
 
 devtools-all: $(DEVTOOLS)
 
-devtools/gen_print_wire.h: $(WIRE_GEN) wire/gen_peer_wire_csv
-	$(WIRE_GEN) --bolt --printwire --header $@ wire_type < wire/gen_peer_wire_csv > $@
+devtools/gen_print_wire.h: $(DEVTOOL_BOLT_DEPS) wire/gen_peer_wire_csv
+	$(BOLT_GEN) -P --page header $@ wire_type < wire/gen_peer_wire_csv > $@
 
-devtools/gen_print_wire.c: $(WIRE_GEN) wire/gen_peer_wire_csv
-	$(WIRE_GEN) --bolt --printwire ${@:.c=.h} wire_type < wire/gen_peer_wire_csv > $@
+devtools/gen_print_wire.c: $(DEVTOOL_BOLT_DEPS) wire/gen_peer_wire_csv
+	$(BOLT_GEN) -P --page impl ${@:.c=.h} wire_type < wire/gen_peer_wire_csv > $@
 
-devtools/gen_print_onion_wire.h: $(WIRE_GEN) wire/gen_onion_wire_csv
-	$(WIRE_GEN) --bolt --printwire --header $@ onion_type < wire/gen_onion_wire_csv > $@
+devtools/gen_print_onion_wire.h: $(DEVTOOL_BOLT_DEPS) wire/gen_onion_wire_csv
+	$(BOLT_GEN) -P --page header $@ onion_type < wire/gen_onion_wire_csv > $@
 
-devtools/gen_print_onion_wire.c: $(WIRE_GEN) wire/gen_onion_wire_csv
-	$(WIRE_GEN) --bolt --printwire ${@:.c=.h} onion_type < wire/gen_onion_wire_csv > $@
+devtools/gen_print_onion_wire.c: $(DEVTOOL_BOLT_DEPS) wire/gen_onion_wire_csv
+	$(BOLT_GEN) -P --page impl ${@:.c=.h} onion_type < wire/gen_onion_wire_csv > $@
 
 devtools/bolt11-cli: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/bolt11-cli.o
 

doc/HACKING.md

@@ -44,8 +44,10 @@ Here's a list of parts, with notes:
 * tools/ - tools for building
   - check-bolt.c: check the source code contains correct BOLT quotes
     (as used by check-source)
-  - generate-wire.py: generate marshal/unmarshal routines from
-    extracts from BOLT specs, and as specified by subdaemons.
+  - generate-wire.py: generates wire marshal/unmarshal-ing
+    routines for subdaemons.
+  - generate-bolts.py: generate marshal/unmarshal routines
+    from extracted BOLT specs.
   - mockup.sh / update-mocks.sh: tools to generate mock functions for
     unit tests.
 
@@ -59,7 +61,7 @@ Here's a list of parts, with notes:
 
 * contrib/ - python support and other stuff which doesn't belong :)
 
-* wire/ - basic marshalling/un
+* wire/ - basic marshalling/un for messages defined in the BOLTs
 
 * common/ - routines needed by any two or more of the directories below
 
@@ -216,6 +218,15 @@ There are three kinds of tests:
 Our Travis CI instance (see `.travis.yml`) runs all these for each
 pull request.
 
+
+Making BOLT Modifications
+-------------------------
+
+All of code for marshalling/unmarshalling BOLT protocol messages is generated
+directly from the spec. These are pegged to the BOLTVERSION, as specified in
+`Makefile`.
+
+
 Source code analysis
 --------------------
 An updated version of the NCC source code analysis tool is available at

gossipd/gossip_constants.h

@@ -7,9 +7,9 @@
  *...
  * 1. type: `onion_packet`
  * 2. data:
- *    * [`1`:`version`]
- *    * [`33`:`public_key`]
- *    * [`20*65`:`hops_data`]
+ *    * [`byte`:`version`]
+ *    * [`point`:`public_key`]
+ *    * [`1300*byte`:`hops_data`]
  */
 #define ROUTING_MAX_HOPS 20
 

gossipd/gossipd.c

@@ -587,9 +587,9 @@ static bool query_short_channel_ids(struct daemon *daemon,
 	 *
 	 * 1. type: 261 (`query_short_channel_ids`) (`gossip_queries`)
 	 * 2. data:
-	 *     * [`32`:`chain_hash`]
-	 *     * [`2`:`len`]
-	 *     * [`len`:`encoded_short_ids`]
+	 *     * [`chain_hash`:`chain_hash`]
+	 *     * [`u16`:`len`]
+	 *     * [`len*byte`:`encoded_short_ids`]
 	 */
 	const size_t reply_overhead = 32 + 2;
 	const size_t max_encoded_bytes = 65535 - 2 - reply_overhead;
@@ -841,12 +841,12 @@ static bool queue_channel_ranges(struct peer *peer,
 	 *
 	 * 1. type: 264 (`reply_channel_range`) (`gossip_queries`)
 	 * 2. data:
-	 *   * [`32`:`chain_hash`]
-	 *   * [`4`:`first_blocknum`]
-	 *   * [`4`:`number_of_blocks`]
-	 *   * [`1`:`complete`]
-	 *   * [`2`:`len`]
-	 *   * [`len`:`encoded_short_ids`]
+	 *   * [`chain_hash`:`chain_hash`]
+	 *   * [`u32`:`first_blocknum`]
+	 *   * [`u32`:`number_of_blocks`]
+	 *   * [`byte`:`complete`]
+	 *   * [`u16`:`len`]
+	 *   * [`len*byte`:`encoded_short_ids`]
 	 */
 	const size_t reply_overhead = 32 + 4 + 4 + 1 + 2;
 	const size_t max_encoded_bytes = 65535 - 2 - reply_overhead;

lightningd/options.c

@@ -252,7 +252,7 @@ static char *opt_set_alias(const char *arg, struct lightningd *ld)
 	ld->alias = tal_free(ld->alias);
 	/* BOLT #7:
 	 *
-	 *    * [`32`:`alias`]
+	 *    * [`32*byte`:`alias`]
 	 *...
 	 *  - MUST set `alias` to a valid UTF-8 string, with any
 	 *   `alias` trailing-bytes equal to 0.

lightningd/peer_htlcs.c

@@ -282,7 +282,7 @@ static void handle_localpay(struct htlc_in *hin,
 	 *
 	 * 1. type: 19 (`final_incorrect_htlc_amount`)
 	 * 2. data:
-	 *    * [`8`:`incoming_htlc_amt`]
+	 *    * [`u64`:`incoming_htlc_amt`]
 	 *
 	 * The amount in the HTLC doesn't match the value in the onion.
 	 */
@@ -295,7 +295,7 @@ static void handle_localpay(struct htlc_in *hin,
 	 *
 	 * 1. type: 18 (`final_incorrect_cltv_expiry`)
 	 * 2. data:
-	 *    * [`4`:`cltv_expiry`]
+	 *    * [`u32`:`cltv_expiry`]
 	 *
 	 * The CLTV expiry in the HTLC doesn't match the value in the onion.
 	 */

lightningd/ping.c

@@ -104,9 +104,9 @@ static struct command_result *json_ping(struct command *cmd,
 	 *...
 	 * 1. type: 18 (`ping`)
 	 * 2. data:
-	 *    * [`2`:`num_pong_bytes`]
-	 *    * [`2`:`byteslen`]
-	 *    * [`byteslen`:`ignored`]
+	 *    * [`u16`:`num_pong_bytes`]
+	 *    * [`u16`:`byteslen`]
+	 *    * [`byteslen*byte`:`ignored`]
 	 */
 	if (*len > 65535 - 2 - 2 - 2) {
 		return command_fail(cmd, JSONRPC2_INVALID_PARAMS,

wire/Makefile

@@ -28,16 +28,18 @@ ALL_GEN_HEADERS += $(WIRE_GEN_HEADERS)
 # They may not have the bolts.
 BOLT_EXTRACT=$(LOCAL_BOLTDIR)/tools/extract-formats.py
 
+WIRE_BOLT_DEPS := $(BOLT_DEPS) tools/gen/impl_template tools/gen/header_template
+
 # Explicit command to re-extract CSV from BOLTs.
 extract-bolt-csv: bolt-precheck
-	@$(BOLT_EXTRACT) --message-fields --message-types $(LOCAL_BOLTDIR)/0[127]*.md > wire/extracted_peer_wire_csv
+	@$(BOLT_EXTRACT) $(LOCAL_BOLTDIR)/0[127]*.md > wire/extracted_peer_wire_csv
 	@echo '#include <wire/onion_defs.h>' > wire/extracted_onion_wire_csv
-	@$(BOLT_EXTRACT) --message-fields --message-types $(LOCAL_BOLTDIR)/04*.md >> wire/extracted_onion_wire_csv
+	@$(BOLT_EXTRACT) $(LOCAL_BOLTDIR)/04*.md >> wire/extracted_onion_wire_csv
 
 # Explicit command to add diff of BOLT CSV's to an experimental_csv file
 extract-experimental-bolt-csv:
-	@$(BOLT_EXTRACT) --message-fields --message-types $(LOCAL_BOLTDIR)/0[127]*.md | diff -u wire/extracted_peer_wire_csv - >>wire/extracted_peer_experimental_csv | if [ $$? -lt 0 ];then exit 1;fi
-	@{ echo '#include <wire/onion_defs.h>'; $(BOLT_EXTRACT) --message-fields --message-types $(LOCAL_BOLTDIR)/04*.md; } | diff -u wire/extracted_onion_wire_csv - >>wire/extracted_onion_experimental_csv | if [ $$? -lt 0 ];then exit 1;fi
+	@$(BOLT_EXTRACT) $(LOCAL_BOLTDIR)/0[127]*.md | diff -u wire/extracted_peer_wire_csv - >>wire/extracted_peer_experimental_csv | if [ $$? -lt 0 ];then exit 1;fi
+	@{ echo '#include <wire/onion_defs.h>'; $(BOLT_EXTRACT) $(LOCAL_BOLTDIR)/04*.md; } | diff -u wire/extracted_onion_wire_csv - >>wire/extracted_onion_experimental_csv | if [ $$? -lt 0 ];then exit 1;fi
 
 wire/extracted_peer_experimental_csv:
 	@touch $@
@@ -60,17 +62,17 @@ wire/gen_onion_wire_csv: wire/extracted_onion_wire_csv
 	@cp $< $@
 endif
 
-wire/gen_peer_wire.h: wire/gen_peer_wire_csv $(WIRE_GEN)
-	$(WIRE_GEN) --bolt --header $@ wire_type < $< > $@
+wire/gen_peer_wire.h: wire/gen_peer_wire_csv $(WIRE_BOLT_DEPS)
+	$(BOLT_GEN) -s --page header $@ wire_type < $< > $@
 
-wire/gen_peer_wire.c: wire/gen_peer_wire_csv $(WIRE_GEN)
-	$(WIRE_GEN) --bolt ${@:.c=.h} wire_type < $< > $@
+wire/gen_peer_wire.c: wire/gen_peer_wire_csv $(WIRE_BOLT_DEPS)
+	$(BOLT_GEN) --page impl ${@:.c=.h} wire_type < $< > $@
 
-wire/gen_onion_wire.h: wire/gen_onion_wire_csv $(WIRE_GEN)
-	$(WIRE_GEN) --bolt --header $@ onion_type < $< > $@
+wire/gen_onion_wire.h: wire/gen_onion_wire_csv $(WIRE_BOLT_DEPS)
+	$(BOLT_GEN) -s --page header $@ onion_type < $< > $@
 
-wire/gen_onion_wire.c: wire/gen_onion_wire_csv $(WIRE_GEN)
-	$(WIRE_GEN) --bolt ${@:.c=.h} onion_type < $< > $@
+wire/gen_onion_wire.c: wire/gen_onion_wire_csv $(WIRE_BOLT_DEPS)
+	$(BOLT_GEN) --page impl ${@:.c=.h} onion_type < $< > $@
 
 check-source: $(WIRE_SRC:%=check-src-include-order/%) $(WIRE_HEADERS_NOGEN:%=check-hdr-include-order/%)