libucl: import 0.9 version (+ fixes upstreamed)

This in particular fixes streamline json export needed for groups

external/libucl/CMakeLists.txt

@@ -1,9 +1,9 @@
-PROJECT(libucl C)
 CMAKE_MINIMUM_REQUIRED(VERSION 3.1.0 FATAL_ERROR)
+PROJECT(libucl C)
 
 SET(LIBUCL_VERSION_MAJOR 0)
-SET(LIBUCL_VERSION_MINOR 8)
-SET(LIBUCL_VERSION_PATCH 3)
+SET(LIBUCL_VERSION_MINOR 9)
+SET(LIBUCL_VERSION_PATCH 0)
 
 SET(LIBUCL_VERSION
         "${LIBUCL_VERSION_MAJOR}.${LIBUCL_VERSION_MINOR}.${LIBUCL_VERSION_PATCH}")

external/libucl/ChangeLog.md

@@ -1,5 +1,21 @@
 # Version history
 
+## Libucl 0.9.0
+
+* 803b588 Breaking: Try to fix streamline embedding
+* 9eddef0 Fix: set p to endptr before checking
+* 25d3f51 Fix broken tests
+* ac644e2 Update makefile.yml
+* 0a5739e Create makefile.yml
+* 987389a Merge branch 'master' into vstakhov-gh-actions
+* 7433904 Import lua code from Rspamd
+* 3912614 Create cmake-multi-platform.yml
+* 3a04c92 lua: Push string with len
+* 2fefed6 Use `_WIN32` instead of `_MSC_VER`
+* aecf17e Avoid build failure trying to create setup.py link if it already exists.
+* 4ef9e6d Add inttypes.h for PRId64
+* dcb43f0 Fix excessive escaping when using ucl_object_fromstring()
+
 ## Libucl 0.5
 
 - Streamline emitter has been added, so it is now possible to output partial `ucl` objects

external/libucl/Makefile.unix

@@ -3,8 +3,8 @@ DESTDIR ?= /usr/local
 LD ?= gcc
 C_COMMON_FLAGS ?= -fPIC -Wall -W -Wno-unused-parameter -Wno-pointer-sign -I./include -I./uthash -I./src -I./klib
 MAJOR_VERSION = 0
-MINOR_VERSION = 2
-PATCH_VERSION = 9
+MINOR_VERSION = 9
+PATCH_VERSION = 0
 VERSION = "$(MAJOR_VERSION).$(MINOR_VERSION).$(PATCH_VERSION)"
 SONAME = libucl.so
 SONAME_FULL = $(SONAME).$(MAJOR_VERSION)

external/libucl/README.md

@@ -1,4 +1,4 @@
-# LIBUCL
+# UCL - Universal Configuration Language
 
 **Table of Contents**  *generated with [DocToc](http://doctoc.herokuapp.com/)*
 
@@ -22,15 +22,19 @@
 
 ## Introduction
 
-This document describes the main features and principles of the configuration
-language called `UCL` - universal configuration language.
+This repository provides the `C` library for parsing configurations written in `UCL` - universal configuration language. It also provides functions to operate with other formats:
+
+* `JSON`: read, write and pretty format
+* `Messagepack`: read and write
+* `S-Expressions`: read only (canonical form)
+* `Yaml`: limited write support (mainly for compatibility)
 
 If you are looking for the libucl API documentation you can find it at [this page](doc/api.md).
 
 ## Basic structure
 
-UCL is heavily infused by `nginx` configuration as the example of a convenient configuration
-system. However, UCL is fully compatible with `JSON` format and is able to parse json files.
+`UCL` is heavily infused by `nginx` configuration as the example of a convenient configuration
+system. However, `UCL` is fully compatible with `JSON` format and is able to parse json files.
 For example, you can write the same configuration in the following ways:
 
 * in nginx like:
@@ -230,7 +234,7 @@ UCL supports external macros both multiline and single line ones:
 ```
 
 Moreover, each macro can accept an optional list of arguments in braces. These
-arguments themselves are the UCL object that is parsed and passed to a macro as
+arguments themselves are the `UCL` object that is parsed and passed to a macro as
 options:
 
 ```nginx

external/libucl/configure.ac

@@ -1,7 +1,7 @@
 m4_define([maj_ver], [0])
-m4_define([med_ver], [8])
-m4_define([min_ver], [3])
-m4_define([so_version], [6:0:1])
+m4_define([med_ver], [9])
+m4_define([min_ver], [0])
+m4_define([so_version], [9:0:0])
 m4_define([ucl_version], [maj_ver.med_ver.min_ver])
 
 AC_INIT([libucl],[ucl_version],[https://github.com/vstakhov/libucl],[libucl])

external/libucl/include/ucl.h

@@ -1411,13 +1411,13 @@ struct ucl_emitter_operations {
 		const ucl_object_t *obj, bool first, bool print_key);
 	/** Start ucl object */
 	void (*ucl_emitter_start_object) (struct ucl_emitter_context *ctx,
-		const ucl_object_t *obj, bool print_key);
+		const ucl_object_t *obj, bool first, bool print_key);
 	/** End ucl object */
 	void (*ucl_emitter_end_object) (struct ucl_emitter_context *ctx,
 		const ucl_object_t *obj);
 	/** Start ucl array */
 	void (*ucl_emitter_start_array) (struct ucl_emitter_context *ctx,
-		const ucl_object_t *obj, bool print_key);
+		const ucl_object_t *obj, bool first, bool print_key);
 	void (*ucl_emitter_end_array) (struct ucl_emitter_context *ctx,
 		const ucl_object_t *obj);
 };

external/libucl/lua/lua_ucl.c

@@ -82,6 +82,11 @@ static ucl_object_t* ucl_object_lua_fromelt (lua_State *L, int idx, ucl_string_f
 
 static void *ucl_null;
 
+struct _rspamd_lua_text {
+	const char *start;
+	unsigned int len;
+	unsigned int flags;
+};
 
 enum lua_ucl_push_flags {
 	LUA_UCL_DEFAULT_FLAGS = 0,
@@ -240,7 +245,7 @@ ucl_object_lua_push_scalar (lua_State *L, const ucl_object_t *obj,
 		lua_pushboolean (L, ucl_obj_toboolean (obj));
 		break;
 	case UCL_STRING:
-		lua_pushstring (L, ucl_obj_tostring (obj));
+		lua_pushlstring (L, ucl_obj_tostring (obj), obj->len);
 		break;
 	case UCL_INT:
 #if LUA_VERSION_NUM >= 501
@@ -510,6 +515,24 @@ ucl_object_lua_fromelt (lua_State *L, int idx, ucl_string_flags_t flags)
 		if (lua_topointer (L, idx) == ucl_null) {
 			obj = ucl_object_typed_new (UCL_NULL);
 		}
+		else {
+			/* Assume it is a text like object */
+			struct _rspamd_lua_text *t = lua_touserdata (L, idx);
+
+			if (t) {
+				if (t->len >0) {
+					obj = ucl_object_fromstring_common(t->start, t->len, 0);
+				}
+				else {
+					obj = ucl_object_fromstring_common("", 0, 0);
+				}
+
+				/* Binary text */
+				if (t->flags & (1u << 5u)) {
+					obj->flags |= UCL_OBJECT_BINARY;
+				}
+			}
+		}
 		break;
 	case LUA_TTABLE:
 	case LUA_TFUNCTION:
@@ -565,10 +588,10 @@ ucl_object_lua_import (lua_State *L, int idx)
 	t = lua_type (L, idx);
 	switch (t) {
 	case LUA_TTABLE:
-		obj = ucl_object_lua_fromtable (L, idx, 0);
+		obj = ucl_object_lua_fromtable (L, idx, UCL_STRING_RAW);
 		break;
 	default:
-		obj = ucl_object_lua_fromelt (L, idx, 0);
+		obj = ucl_object_lua_fromelt (L, idx, UCL_STRING_RAW);
 		break;
 	}
 
@@ -593,10 +616,10 @@ ucl_object_lua_import_escape (lua_State *L, int idx)
 	t = lua_type (L, idx);
 	switch (t) {
 	case LUA_TTABLE:
-		obj = ucl_object_lua_fromtable (L, idx, UCL_STRING_RAW);
+		obj = ucl_object_lua_fromtable (L, idx, UCL_STRING_ESCAPE);
 		break;
 	default:
-		obj = ucl_object_lua_fromelt (L, idx, UCL_STRING_RAW);
+		obj = ucl_object_lua_fromelt (L, idx, UCL_STRING_ESCAPE);
 		break;
 	}
 
@@ -607,11 +630,12 @@ static int
 lua_ucl_to_string (lua_State *L, const ucl_object_t *obj, enum ucl_emitter type)
 {
 	unsigned char *result;
+	size_t outlen;
 
-	result = ucl_object_emit (obj, type);
+	result = ucl_object_emit_len (obj, type, &outlen);
 
 	if (result != NULL) {
-		lua_pushstring (L, (const char *)result);
+		lua_pushlstring (L, (const char *)result, outlen);
 		free (result);
 	}
 	else {
@@ -634,7 +658,6 @@ lua_ucl_parser_init (lua_State *L)
 	parser = ucl_parser_new (flags);
 	if (parser == NULL) {
 		lua_pushnil (L);
-		return 1;
 	}
 
 	pparser = lua_newuserdata (L, sizeof (parser));
@@ -843,12 +866,6 @@ lua_ucl_parser_parse_string (lua_State *L)
 	return ret;
 }
 
-struct _rspamd_lua_text {
-	const char *start;
-	unsigned int len;
-	unsigned int flags;
-};
-
 /***
  * @method parser:parse_text(input)
  * Parse UCL object from text object (Rspamd specific).
@@ -864,7 +881,24 @@ lua_ucl_parser_parse_text (lua_State *L)
 	int ret = 2;
 
 	parser = lua_ucl_parser_get (L, 1);
-	t = lua_touserdata (L, 2);
+
+	if (lua_type (L, 2) == LUA_TUSERDATA) {
+		t = lua_touserdata (L, 2);
+	}
+	else if (lua_type (L, 2) == LUA_TSTRING) {
+		const gchar *s;
+		gsize len;
+		static struct _rspamd_lua_text st_t;
+
+		s = lua_tolstring (L, 2, &len);
+		st_t.start = s;
+		st_t.len = len;
+
+		t = &st_t;
+	}
+	else {
+		return luaL_error(L, "invalid argument as input, expected userdata or a string");
+	}
 
 	if (lua_type (L, 3) == LUA_TSTRING) {
 		type = lua_ucl_str_to_parse_type (lua_tostring (L, 3));
@@ -1435,10 +1469,11 @@ lua_ucl_to_format (lua_State *L)
 				format = UCL_EMIT_YAML;
 			}
 			else if (strcasecmp (strtype, "config") == 0 ||
-				strcasecmp (strtype, "ucl") == 0) {
+					 strcasecmp (strtype, "ucl") == 0) {
 				format = UCL_EMIT_CONFIG;
 			}
-			else if (strcasecmp (strtype, "msgpack") == 0) {
+			else if (strcasecmp (strtype, "msgpack") == 0 ||
+					 strcasecmp (strtype, "messagepack") == 0) {
 				format = UCL_EMIT_MSGPACK;
 			}
 		}