ccan: import ccan/json_out and ccan/json_escape.

These are generalized from our internal implementations.

The main difference is that 'struct json_escaped' is now 'struct
json_escape', so we replace that immediately.

The difference between lightningd's json-writing ringbuffer and the
more generic ccan/json_out is that the latter has a better API and
handles escaping transparently if something slips through (though
it does offer direct accessors so you can mess things up yourself!).

Signed-off-by: Rusty Russell <[email protected]>

Makefile

@@ -31,7 +31,7 @@ SANITIZER_FLAGS=
 endif
 
 ifeq ($(DEVELOPER),1)
-DEV_CFLAGS=-DCCAN_TAKE_DEBUG=1 -DCCAN_TAL_DEBUG=1
+DEV_CFLAGS=-DCCAN_TAKE_DEBUG=1 -DCCAN_TAL_DEBUG=1 -DCCAN_JSON_OUT_DEBUG=1
 else
 DEV_CFLAGS=
 endif
@@ -81,6 +81,8 @@ CCAN_OBJS :=					\
 	ccan-io-fdpass.o			\
 	ccan-isaac.o				\
 	ccan-isaac64.o				\
+	ccan-json_escape.o			\
+	ccan-json_out.o				\
 	ccan-list.o				\
 	ccan-mem.o				\
 	ccan-membuf.o				\
@@ -144,6 +146,8 @@ CCAN_HEADERS :=						\
 	$(CCANDIR)/ccan/io/io_plan.h			\
 	$(CCANDIR)/ccan/isaac/isaac.h			\
 	$(CCANDIR)/ccan/isaac/isaac64.h			\
+	$(CCANDIR)/ccan/json_escape/json_escape.h	\
+	$(CCANDIR)/ccan/json_out/json_out.h		\
 	$(CCANDIR)/ccan/likely/likely.h			\
 	$(CCANDIR)/ccan/list/list.h			\
 	$(CCANDIR)/ccan/mem/mem.h			\
@@ -657,3 +661,7 @@ ccan-bitmap.o: $(CCANDIR)/ccan/bitmap/bitmap.c
 	$(CC) $(CFLAGS) -c -o $@ $<
 ccan-membuf.o: $(CCANDIR)/ccan/membuf/membuf.c
 	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-json_escape.o: $(CCANDIR)/ccan/json_escape/json_escape.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-json_out.o: $(CCANDIR)/ccan/json_out/json_out.c
+	$(CC) $(CFLAGS) -c -o $@ $<

ccan/ccan/json_escape/LICENSE

@@ -0,0 +1 @@
+../../licenses/BSD-MIT

ccan/ccan/json_escape/_info

@@ -0,0 +1,39 @@
+#include "config.h"
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * json_escape - Escape sequences for JSON strings
+ *
+ * This code helps you format strings into forms useful for JSON.
+ *
+ * Author: Rusty Russell <[email protected]>
+ * License: BSD-MIT
+ * Example:
+ *	// Print arguments as a JSON array.
+ *	#include <ccan/json_escape/json_escape.h>
+ *
+ *	int main(int argc, char *argv[])
+ *	{
+ *		printf("[");
+ *		for (int i = 1; i < argc; i++) {
+ *			struct json_escape *e = json_escape(NULL, argv[i]);
+ *			printf("%s\"%s\"", i == 1 ? "" : ",", e->s);
+ *		}
+ *		printf("]\n");
+ *		return 0;
+ *	}
+ */
+int main(int argc, char *argv[])
+{
+	/* Expect exactly one argument */
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/tal\n");
+		return 0;
+	}
+
+	return 1;
+}

common/json_escaped.c → ccan/ccan/json_escape/json_escape.c

@@ -1,46 +1,58 @@
-#include <common/json_escaped.h>
+/* MIT (BSD) license - see LICENSE file for details */
+#include <ccan/json_escape/json_escape.h>
 #include <stdio.h>
 
-struct json_escaped *json_escaped_string_(const tal_t *ctx,
-					  const void *bytes, size_t len)
+struct json_escape *json_escape_string_(const tal_t *ctx,
+					const void *bytes, size_t len)
 {
-	struct json_escaped *esc;
+	struct json_escape *esc;
 
 	esc = (void *)tal_arr_label(ctx, char, len + 1,
-				    TAL_LABEL(struct json_escaped, ""));
+				    TAL_LABEL(struct json_escape, ""));
 	memcpy(esc->s, bytes, len);
 	esc->s[len] = '\0';
 	return esc;
 }
 
-struct json_escaped *json_to_escaped_string(const tal_t *ctx,
-					    const char *buffer,
-					    const jsmntok_t *tok)
+bool json_escape_eq(const struct json_escape *a, const struct json_escape *b)
 {
-	if (tok->type != JSMN_STRING)
-		return NULL;
-	/* jsmn always gives us ~ well-formed strings. */
-	return json_escaped_string_(ctx, buffer + tok->start,
-				    tok->end - tok->start);
+	return streq(a->s, b->s);
 }
 
-bool json_escaped_eq(const struct json_escaped *a,
-		     const struct json_escaped *b)
+bool json_escape_needed(const char *str, size_t len)
 {
-	return streq(a->s, b->s);
+	for (size_t i = 0; i < len; i++) {
+		if ((unsigned)str[i] < ' '
+		    || str[i] == 127
+		    || str[i] == '"'
+		    || str[i] == '\\')
+			return true;
+	}
+	return false;
 }
 
-static struct json_escaped *escape(const tal_t *ctx,
-				   const char *str TAKES,
-				   bool partial)
+static struct json_escape *escape(const tal_t *ctx,
+				  const char *str TAKES,
+				  size_t len,
+				  bool partial)
 {
-	struct json_escaped *esc;
+	struct json_escape *esc;
 	size_t i, n;
 
+	/* Fast path: can steal, and nothing to escape. */
+	if (is_taken(str)
+	    && tal_count(str) > len
+	    && !json_escape_needed(str, len)) {
+		taken(str);
+		esc = (struct json_escape *)tal_steal(ctx, str);
+		esc->s[len] = '\0';
+		return esc;
+	}
+
 	/* Worst case: all \uXXXX */
-	esc = (struct json_escaped *)tal_arr(ctx, char, strlen(str) * 6 + 1);
+	esc = (struct json_escape *)tal_arr(ctx, char, len * 6 + 1);
 
-	for (i = n = 0; str[i]; i++, n++) {
+	for (i = n = 0; i < len; i++, n++) {
 		char escape = 0;
 		switch (str[i]) {
 		case '\n':
@@ -107,19 +119,24 @@ static struct json_escaped *escape(const tal_t *ctx,
 	return esc;
 }
 
-struct json_escaped *json_partial_escape(const tal_t *ctx, const char *str TAKES)
+struct json_escape *json_partial_escape(const tal_t *ctx, const char *str TAKES)
+{
+	return escape(ctx, str, strlen(str), true);
+}
+
+struct json_escape *json_escape(const tal_t *ctx, const char *str TAKES)
 {
-	return escape(ctx, str, true);
+	return escape(ctx, str, strlen(str), false);
 }
 
-struct json_escaped *json_escape(const tal_t *ctx, const char *str TAKES)
+struct json_escape *json_escape_len(const tal_t *ctx, const char *str TAKES,
+				    size_t len)
 {
-	return escape(ctx, str, false);
+	return escape(ctx, str, len, false);
 }
 
 /* By policy, we don't handle \u.  Use UTF-8. */
-const char *json_escaped_unescape(const tal_t *ctx,
-				  const struct json_escaped *esc)
+const char *json_escape_unescape(const tal_t *ctx, const struct json_escape *esc)
 {
 	char *unesc = tal_arr(ctx, char, strlen(esc->s) + 1);
 	size_t i, n;

ccan/ccan/json_escape/json_escape.h

@@ -0,0 +1,44 @@
+/* MIT (BSD) license - see LICENSE file for details */
+#ifndef CCAN_JSON_ESCAPE_H
+#define CCAN_JSON_ESCAPE_H
+#include "config.h"
+#include <ccan/tal/tal.h>
+
+/* Type differentiation for a correctly-escaped JSON string */
+struct json_escape {
+	/* NUL terminated string. */
+	char s[1];
+};
+
+/**
+ * json_escape - escape a valid UTF-8 string.
+ * @ctx: tal context to allocate from.
+ * @str: the string to escape.
+ *
+ * Allocates and returns a valid JSON string (without surrounding quotes).
+ */
+struct json_escape *json_escape(const tal_t *ctx, const char *str TAKES);
+
+/* Version with @len */
+struct json_escape *json_escape_len(const tal_t *ctx,
+				    const char *str TAKES, size_t len);
+
+/* @str is a valid UTF-8 string which may already contain escapes. */
+struct json_escape *json_partial_escape(const tal_t *ctx,
+					 const char *str TAKES);
+
+/* Do we need to escape this str? */
+bool json_escape_needed(const char *str, size_t len);
+
+/* Are two escape json strings identical? */
+bool json_escape_eq(const struct json_escape *a,
+		     const struct json_escape *b);
+
+/* Internal routine for creating json_escape from bytes. */
+struct json_escape *json_escape_string_(const tal_t *ctx,
+					const void *bytes, size_t len);
+
+/* Be very careful here!  Can fail!  Doesn't handle \u: use UTF-8 please. */
+const char *json_escape_unescape(const tal_t *ctx,
+				 const struct json_escape *esc);
+#endif /* CCAN_JSON_ESCAPE_H */

ccan/ccan/json_escape/test/run-partial.c

@@ -0,0 +1,41 @@
+#include <ccan/json_escape/json_escape.h>
+/* Include the C files directly. */
+#include <ccan/json_escape/json_escape.c>
+#include <ccan/tap/tap.h>
+
+int main(void)
+{
+	const tal_t *ctx = tal(NULL, char);
+
+	/* This is how many tests you plan to run */
+	plan_tests(21);
+
+	ok1(!strcmp(json_partial_escape(ctx, "\\")->s, "\\\\"));
+	ok1(!strcmp(json_partial_escape(ctx, "\\\\")->s, "\\\\"));
+	ok1(!strcmp(json_partial_escape(ctx, "\\\\\\")->s, "\\\\\\\\"));
+	ok1(!strcmp(json_partial_escape(ctx, "\\\\\\\\")->s, "\\\\\\\\"));
+	ok1(!strcmp(json_partial_escape(ctx, "\\n")->s, "\\n"));
+	ok1(!strcmp(json_partial_escape(ctx, "\n")->s, "\\n"));
+	ok1(!strcmp(json_partial_escape(ctx, "\\\"")->s, "\\\""));
+	ok1(!strcmp(json_partial_escape(ctx, "\"")->s, "\\\""));
+	ok1(!strcmp(json_partial_escape(ctx, "\\t")->s, "\\t"));
+	ok1(!strcmp(json_partial_escape(ctx, "\t")->s, "\\t"));
+	ok1(!strcmp(json_partial_escape(ctx, "\\b")->s, "\\b"));
+	ok1(!strcmp(json_partial_escape(ctx, "\b")->s, "\\b"));
+	ok1(!strcmp(json_partial_escape(ctx, "\\r")->s, "\\r"));
+	ok1(!strcmp(json_partial_escape(ctx, "\r")->s, "\\r"));
+	ok1(!strcmp(json_partial_escape(ctx, "\\f")->s, "\\f"));
+	ok1(!strcmp(json_partial_escape(ctx, "\f")->s, "\\f"));
+	/* You're allowed to escape / according to json.org. */
+	ok1(!strcmp(json_partial_escape(ctx, "\\/")->s, "\\/"));
+	ok1(!strcmp(json_partial_escape(ctx, "/")->s, "/"));
+
+	ok1(!strcmp(json_partial_escape(ctx, "\\u0FFF")->s, "\\u0FFF"));
+	ok1(!strcmp(json_partial_escape(ctx, "\\u0FFFx")->s, "\\u0FFFx"));
+
+	/* Unknown escapes should be escaped. */
+	ok1(!strcmp(json_partial_escape(ctx, "\\x")->s, "\\\\x"));
+	tal_free(ctx);
+
+	return 0;
+}

ccan/ccan/json_escape/test/run-take.c

@@ -0,0 +1,35 @@
+#include <ccan/json_escape/json_escape.h>
+/* Include the C files directly. */
+#include <ccan/json_escape/json_escape.c>
+#include <ccan/tap/tap.h>
+
+int main(void)
+{
+	const tal_t *ctx = tal(NULL, char);
+	struct json_escape *e;
+	char *p;
+
+	/* This is how many tests you plan to run */
+	plan_tests(5);
+
+	/* This should simply be tal_steal */
+	p = tal_dup_arr(NULL, char, "Hello", 6, 0);
+	e = json_escape(ctx, take(p));
+	ok1(!strcmp(e->s, "Hello"));
+	ok1((void *)e == (void *)p);
+	ok1(tal_parent(e) == ctx);
+
+	/* This can't be tal_steal, but still should be freed. */
+	p = tal_dup_arr(NULL, char,
+			"\\\b\f\n\r\t\""
+			"\\\\\\b\\f\\n\\r\\t\\\"", 22, 0);
+	e = json_escape(ctx, take(p));
+	ok1(tal_parent(e) == ctx);
+	ok1(!strcmp(e->s,
+		    "\\\\\\b\\f\\n\\r\\t\\\""
+		    "\\\\\\\\\\\\b\\\\f\\\\n\\\\r\\\\t\\\\\\\""));
+	tal_free(ctx);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/json_escape/test/run.c

@@ -0,0 +1,44 @@
+#include <ccan/json_escape/json_escape.h>
+/* Include the C files directly. */
+#include <ccan/json_escape/json_escape.c>
+#include <ccan/tap/tap.h>
+
+int main(void)
+{
+	const tal_t *ctx = tal(NULL, char);
+	struct json_escape *e;
+
+	/* This is how many tests you plan to run */
+	plan_tests(6);
+
+	e = json_escape(ctx, "Hello");
+	ok1(!strcmp(e->s, "Hello"));
+	ok1(!strcmp(json_escape_unescape(ctx, e),
+		    "Hello"));
+
+	e = json_escape(ctx,
+			"\\\b\f\n\r\t\""
+			"\\\\\\b\\f\\n\\r\\t\\\"");
+	ok1(!strcmp(e->s,
+		    "\\\\\\b\\f\\n\\r\\t\\\""
+		    "\\\\\\\\\\\\b\\\\f\\\\n\\\\r\\\\t\\\\\\\""));
+	ok1(!strcmp(json_escape_unescape(ctx, e),
+		    "\\\b\f\n\r\t\""
+		    "\\\\\\b\\f\\n\\r\\t\\\""));
+
+	/* This one doesn't escape the already-escaped chars */
+	e = json_partial_escape(ctx,
+				"\\\b\f\n\r\t\""
+				"\\\\\\b\\f\\n\\r\\t\\\"");
+	ok1(!strcmp(e->s,
+		    "\\\\\\b\\f\\n\\r\\t\\\""
+		    "\\\\\\b\\f\\n\\r\\t\\\""));
+	ok1(!strcmp(json_escape_unescape(ctx, e),
+		    "\\\b\f\n\r\t\""
+		    "\\\b\f\n\r\t\""));
+
+	tal_free(ctx);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/json_out/LICENSE

@@ -0,0 +1 @@
+../../licenses/BSD-MIT