tools: add a tool to formatting schema, and fixed (hack) the Makefile…

… code checking.

This include the following commits:

- review 1/2: move from tab to space, and remove the exp. prop from doc;
- review 2/2: remove experimental features;


Signed-off-by: Vincenzo Palazzo <[email protected]>

doc/Makefile

@@ -89,7 +89,15 @@ MANPAGES := doc/lightning-cli.1 \
 	doc/lightning-help.7 \
 	doc/lightning-getlog.7
 
-doc-all: $(MANPAGES) doc/index.rst
+doc-all: fmt-schema $(MANPAGES) doc/index.rst
+
+#FIXME(vincenzopalazzo) we don't need to change a file if it is well format,
+#so we can use diff to check difference in the real json and in the .fmt
+fmt-schema:
+	@echo "Checking schema fmt!"
+	@for f in $$(find doc/schemas -type f -name '*.json'); do cat "$$f" | jq . > "$$f.fmt" ; mv "$$f.fmt" "$$f" ; done
+
+check-doc: check-config-docs check-manpages
 
 # Some manpages use a schema, so need that added.
 MARKDOWN_WITH_SCHEMA := $(shell grep -l GENERATE-FROM-SCHEMA $(MANPAGES:=.md))
@@ -149,7 +157,7 @@ check-manpages: all-programs check-config-docs
 # Makes sure that fields mentioned in schema are in man page, and vice versa.
 check-config-docs:
 	@for c in `sed -n 's/^	"\(.*\)": {/\1/p' doc/schemas/listconfigs.schema.json | grep -v '^# version$$' | grep -v '^plugins$$' | grep -v '^important-plugins$$'`; do if ! grep -q "^ \*\*$$c\*\*" doc/lightningd-config.5.md; then echo "$$c undocumented!"; exit 1; fi; done
-	@for c in `grep -v '\[plugin ' doc/lightningd-config.5.md | sed -n 's/^ \*\*\([^*]*\)\*\*.*/\1/p' | grep -v '^\(help\|version\|mainnet\|testnet\|signet\|plugin\|important-plugin\|plugin-dir\|clear-plugins\)$$'`; do if ! grep -q '^	"'"$$c"'"' doc/schemas/listconfigs.schema.json; then echo "$$c documented but not in schema!"; exit 1; fi; done
+	@for c in `grep -v '\[plugin ' doc/lightningd-config.5.md | sed -n 's/^ \*\*\([^*]*\)\*\*.*/\1/p' | grep -v '^\(help\|version\|mainnet\|testnet\|signet\|plugin\|important-plugin\|plugin-dir\|clear-plugins\)$$'`; do if ! grep -q '"'"$$c"'"' doc/schemas/listconfigs.schema.json; then echo "$$c documented but not in schema!"; exit 1; fi; done
 
 doc-maintainer-clean:
 	$(RM) $(MANPAGES)

doc/schemas/WRITING_SCHEMAS.md

@@ -39,6 +39,9 @@ You should always list all fields which are *always* present in
 We extend the basic types; see
 [contrib/pyln-testing/pyln/testing/fixtures.py](fixtures.py).
 
+In addition, before to commit a new schema or a new version of it, make sure that it
+is well formatted. If you don't want do it by hand, use `make fmt-schema` that uses
+jq under the hood.
 
 ### Using Conditional Fields
 

doc/undoc-flags.json

@@ -0,0 +1,21 @@
+{
+    "flags": [
+        "experimental-accept-extra-tlv-types",
+        "channel-fee-max-base-msat",
+        "channel-fee-max-proportional-thousandths",
+        "funder-fund-probability",
+        "funder-fuzz-percent",
+        "funder-lease-requests-only",
+        "funder-max-their-funding",
+        "funder-min-their-funding",
+        "funder-per-channel-max",
+        "funder-per-channel-min",
+        "funder-policy",
+        "funder-policy-mod",
+        "funder-reserve-tank",
+        "lease-fee-base-msat",
+        "lease-fee-basis",
+        "lease-funding-weight",
+        "fetchinvoice-noconnect"
+    ]
+}

tools/check-manpage.sh

@@ -32,6 +32,16 @@ CMD_OPTNAMES=$(get_cmd_opts "$1" | sort)
 # Now, gather (long) opt names from man page, make sure they match.
 MAN_OPTNAMES=$(sed -E -n 's/^ \*\*(--)?([^*/]*)\*\*(=?).*/\2\3/p' < "$2" | sort)
 
+# Remove undocumented proprieties, usually these proprieties are
+# under experimental phases.
+for flag in $(jq '.flags[]' <doc/undoc-flags.json) ; do
+    # Remove the quotes from the string, so the code will remove
+    # the first and last char in the string.
+    FLAG=$(sed 's/.//;s/.$//' <(echo "$flag"))
+    CMD_OPTNAMES=$(sed "/$FLAG=/d" <(echo "$CMD_OPTNAMES"))
+done
+
+
 if [ "$CMD_OPTNAMES" != "$MAN_OPTNAMES" ]; then
     echo "diff of command names vs manpage names":
     diff -u <(echo "$CMD_OPTNAMES") <(echo "$MAN_OPTNAMES")