Normalize the Avro logical types

Any LDMS type that doesn't have an Avro equivalent is encoded in an Avro type with a logicalType annotation that indicates the type from which the value came.
ovis-hpc · Apr 20, 2023 · a3cc5d7 · a3cc5d7
1 parent 29518af
commit a3cc5d7
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 11 deletions.
diff --git a/ldms/src/ldmsd/ldmsd_decomp.c b/ldms/src/ldmsd/ldmsd_decomp.c
@@ -748,23 +748,23 @@ int ldmsd_row_to_json_object(ldmsd_row_t row, char **str, int *len)
 static const char *col_type_str(enum ldms_value_type type)
 {
 	static char *type_str[] = {
-	    [LDMS_V_CHAR] = "\"string\"",
-	    [LDMS_V_U8] = "\"string\"",
-	    [LDMS_V_S8] = "\"int\"",
+	    [LDMS_V_CHAR] = "{\"type\":\"string\",\"logicalType\":\"single-character\"}",
+	    [LDMS_V_U8] = "{\"type\":\"int\",\"logicalType\":\"uint8\"}",
+	    [LDMS_V_S8] = "{\"type\":\"int\",\"logicalType\":\"int8\"}",
 	    [LDMS_V_U16] = "{\"type\":\"int\",\"logicalType\":\"unsigned-short\"}",
-	    [LDMS_V_S16] = "\"int\"",
-	    [LDMS_V_U32] = "{\"type\":\"int\",\"logicalType\":\"unsigned-short\"}",
+	    [LDMS_V_S16] = "{\"type\":\"int\",\"logicalType\":\"signed-short\"}",
+	    [LDMS_V_U32] = "{\"type\":\"long\",\"logicalType\":\"unsigned-int\"}",
 	    [LDMS_V_S32] = "\"int\"",
-	    [LDMS_V_U64] = "{\"type\":\"int\",\"logicalType\":\"unsigned-short\"}",
+	    [LDMS_V_U64] = "{\"type\":\"long\",\"logicalType\":\"unsigned-long\"}",
 	    [LDMS_V_S64] = "\"long\"",
 	    [LDMS_V_F32] = "\"float\"",
 	    [LDMS_V_D64] = "\"double\"",
 	    [LDMS_V_CHAR_ARRAY] = "\"string\"",
 	    [LDMS_V_U8_ARRAY] = "{\"type\":\"int\",\"logicalType\":\"unsigned-char\"}",
-	    [LDMS_V_S8_ARRAY] = "\"int\"",
+	    [LDMS_V_S8_ARRAY] = "{\"type\":\"int\",\"logicalType\":\"signed-char\"}",
 	    [LDMS_V_U16_ARRAY] = "{\"type\":\"int\",\"logicalType\":\"unsigned-short\"}",
-	    [LDMS_V_S16_ARRAY] = "\"int\"",
-	    [LDMS_V_U32_ARRAY] = "{\"type\":\"int\",\"logicalType\":\"unsigned-int\"}",
+	    [LDMS_V_S16_ARRAY] = "{\"type\":\"int\",\"logicalType\":\"signed-short\"}",
+	    [LDMS_V_U32_ARRAY] = "{\"type\":\"long\",\"logicalType\":\"unsigned-int\"}",
 	    [LDMS_V_S32_ARRAY] = "\"int\"",
 	    [LDMS_V_U64_ARRAY] = "{\"type\":\"long\",\"logicalType\":\"unsigned-long\"}",
 	    [LDMS_V_S64_ARRAY] = "\"long\"",

diff --git a/ldms/src/store/avro_kafka/store_avro_kafka.c b/ldms/src/store/avro_kafka/store_avro_kafka.c
@@ -499,6 +499,7 @@ static int set_avro_value_from_col(avro_value_t *col_value,
 	struct ldms_timestamp ts;
 	avro_value_t val, item_val;
 	long ms;
+	char char_str[2];
 	size_t item_idx;
 	enum avro_type_t t = avro_value_get_type(col_value);
 	switch (col->type) {
@@ -537,8 +538,9 @@ static int set_avro_value_from_col(avro_value_t *col_value,
 		}
 		break;
 	case LDMS_V_CHAR:
-		rc = avro_value_set_int(col_value,
-				(int)ldms_mval_get_char(col->mval));
+		char_str[0] = ldms_mval_get_char(col->mval);
+		char_str[1] = '\0';
+		rc = avro_value_set_string(col_value, char_str);
 		break;
 	case LDMS_V_U8:
 		rc = avro_value_set_int(col_value,