Merge pull request #140 from KevRiver/use-typed-key-values

Use typed key values

cmd/humanlog/query.go

@@ -17,6 +17,7 @@ import (
 	"github.com/charmbracelet/lipgloss"
 	"github.com/charmbracelet/x/term"
 	"github.com/crazy3lf/colorconv"
+	"github.com/humanlogio/api/go/pkg/logql"
 	"github.com/humanlogio/api/go/svc/environment/v1/environmentv1connect"
 	"github.com/humanlogio/api/go/svc/organization/v1/organizationv1connect"
 	queryv1 "github.com/humanlogio/api/go/svc/query/v1"
@@ -330,12 +331,16 @@ func queryApiWatchCmd(
 			if state.CurrentEnvironmentID != nil {
 				environmentID = *state.CurrentEnvironmentID
 			}
+			lq, err := logql.ParseLogQuery(query)
+			if err != nil {
+				return fmt.Errorf("parsing query: %v", err)
+			}
 			req := &queryv1.WatchQueryRequest{
 				EnvironmentId: environmentID,
 				Query: &typesv1.LogQuery{
 					From:  from,
 					To:    to,
-					Query: query,
+					Query: lq.Query,
 				},
 			}
 			res, err := queryClient.WatchQuery(ctx, connect.NewRequest(req))

docker_compose_handler.go

@@ -24,7 +24,7 @@ func tryDockerComposePrefix(d []byte, ev *typesv1.StructuredLogEvent, nextHandle
 	if matches != nil {
 		if nextHandler.TryHandle(matches[2], ev) {
 			ev.Kvs = append(ev.Kvs, &typesv1.KV{
-				Key: "service", Value: string(matches[1]),
+				Key: "service", Value: typesv1.ValStr(string(matches[1])),
 			})
 			return true
 		}
@@ -34,7 +34,7 @@ func tryDockerComposePrefix(d []byte, ev *typesv1.StructuredLogEvent, nextHandle
 		case *JSONHandler:
 			if tryZapDevDCPrefix(matches[2], ev, h) {
 				ev.Kvs = append(ev.Kvs, &typesv1.KV{
-					Key: "service", Value: string(matches[1]),
+					Key: "service", Value: typesv1.ValStr(string(matches[1])),
 				})
 				return true
 			}

go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/go-logfmt/logfmt v0.5.1
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
-	github.com/humanlogio/api/go v0.0.0-20241128170213-590d167300cd
+	github.com/humanlogio/api/go v0.0.0-20241211090836-a1e1ce8a4f72
 	github.com/humanlogio/humanlog-pro v0.0.0-20241129104809-3580d74828a9
 	github.com/kr/logfmt v0.0.0-20210122060352-19f9bcb100e6
 	github.com/lrstanley/bubblezone v0.0.0-20240914071701-b48c55a5e78e

go.sum

@@ -80,8 +80,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU=
 github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0=
-github.com/humanlogio/api/go v0.0.0-20241128170213-590d167300cd h1:449C6cnB4W6DblDMPfCfA4xyEkiYMpngGf7TEX9O8ro=
-github.com/humanlogio/api/go v0.0.0-20241128170213-590d167300cd/go.mod h1:+hU/MU1g6QvtbeknKOlUI1yEStVqkPJ8jmYIj63OV5I=
+github.com/humanlogio/api/go v0.0.0-20241208082433-416862db1fa7 h1:lsiJGrN2E5qW6yvD6TKTAHa8lURmnh6KAuHvRfU8csU=
+github.com/humanlogio/api/go v0.0.0-20241208082433-416862db1fa7/go.mod h1:+hU/MU1g6QvtbeknKOlUI1yEStVqkPJ8jmYIj63OV5I=
+github.com/humanlogio/api/go v0.0.0-20241211090836-a1e1ce8a4f72 h1:68dDinP4+R4eaEaVXOCiZEwypLk3aUEw4gnF7QL+oH0=
+github.com/humanlogio/api/go v0.0.0-20241211090836-a1e1ce8a4f72/go.mod h1:pFt3YKuAVJk5nziOiKXTKyq5fj4aA9azq6xOx/932KQ=
 github.com/humanlogio/humanlog-pro v0.0.0-20241129104809-3580d74828a9 h1:tdUCzFh8qvnWNCmxub0KSj1lIiCeWqvRjsMSSIApneE=
 github.com/humanlogio/humanlog-pro v0.0.0-20241129104809-3580d74828a9/go.mod h1:zq05mTZQXvKheFiAGlPx6+VSo29jw2ER8oy8DIQKW2Q=
 github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=

internal/localsvc/svc.go

@@ -275,7 +275,7 @@ func (svc *Service) WatchQuery(ctx context.Context, req *connect.Request[qrv1.Wa
 	query := req.Msg.GetQuery()
 
 	ll := svc.ll.With(
-		slog.String("query.query", query.Query),
+		slog.String("query.query", query.Query.String()),
 	)
 	if query.From != nil {
 		ll = ll.With(slog.String("query.from", query.From.AsTime().Format(time.RFC3339Nano)))

internal/memstorage/memory.go

@@ -223,7 +223,7 @@ func newMemStorageSink(ll *slog.Logger, id SinkID) *MemStorageSink {
 func (snk *MemStorageSink) queryLogger(q *typesv1.LogQuery) *slog.Logger {
 	ll := snk.ll.With(
 		slog.Bool("sink.closed", snk.closed),
-		slog.String("query", q.Query),
+		slog.String("query", q.Query.String()),
 	)
 	if q.From != nil {
 		ll = ll.With(slog.Time("from", q.From.AsTime()))

json_handler.go

@@ -19,7 +19,7 @@ type JSONHandler struct {
 	Level   string
 	Time    time.Time
 	Message string
-	Fields  map[string]string
+	Fields  map[string]*typesv1.Val
 }
 
 // searchJSON searches a document for a key using the found func to determine if the value is accepted.
@@ -55,7 +55,7 @@ func (h *JSONHandler) clear() {
 	h.Level = ""
 	h.Time = time.Time{}
 	h.Message = ""
-	h.Fields = make(map[string]string)
+	h.Fields = make(map[string]*typesv1.Val)
 }
 
 // TryHandle tells if this line was handled by this handler.
@@ -95,22 +95,24 @@ func deleteJSONKey(key string, jsonData map[string]interface{}) {
 	}
 }
 
-func getFlattenedFields(v map[string]interface{}) map[string]string {
-	extValues := make(map[string]string)
+func getFlattenedFields(v map[string]interface{}) map[string]*typesv1.Val {
+	extValues := make(map[string]*typesv1.Val)
 	for key, nestedVal := range v {
 		switch valTyped := nestedVal.(type) {
 		case json.Number:
 			if z, err := valTyped.Int64(); err == nil {
-				extValues[key] = fmt.Sprintf("%d", z)
+				extValues[key] = typesv1.ValI64(z)
 				continue
 			}
 			if f, err := valTyped.Float64(); err == nil {
-				extValues[key] = fmt.Sprintf("%g", f)
+				extValues[key] = typesv1.ValF64(f)
 				continue
 			}
-			extValues[key] = valTyped.String()
+			extValues[key] = typesv1.ValStr(valTyped.String())
 		case string:
-			extValues[key] = fmt.Sprintf("%q", valTyped)
+			extValues[key] = typesv1.ValStr(valTyped)
+		case bool:
+			extValues[key] = typesv1.ValBool(valTyped)
 		case []interface{}:
 			flattenedArrayFields := getFlattenedArrayFields(valTyped)
 			for k, v := range flattenedArrayFields {
@@ -122,26 +124,28 @@ func getFlattenedFields(v map[string]interface{}) map[string]string {
 				extValues[key+"."+keyNested] = valStr
 			}
 		default:
-			extValues[key] = fmt.Sprintf("%v", valTyped)
+			extValues[key] = typesv1.ValStr(fmt.Sprintf("%v", valTyped))
 		}
 	}
 	return extValues
 }
 
-func getFlattenedArrayFields(data []interface{}) map[string]string {
-	flattened := make(map[string]string)
+func getFlattenedArrayFields(data []interface{}) map[string]*typesv1.Val {
+	flattened := make(map[string]*typesv1.Val)
 	for i, v := range data {
 		switch vt := v.(type) {
 		case json.Number:
 			if z, err := vt.Int64(); err == nil {
-				flattened[strconv.Itoa(i)] = fmt.Sprintf("%d", z)
+				flattened[strconv.Itoa(i)] = typesv1.ValI64(z)
 			} else if f, err := vt.Float64(); err == nil {
-				flattened[strconv.Itoa(i)] = fmt.Sprintf("%g", f)
+				flattened[strconv.Itoa(i)] = typesv1.ValF64(f)
 			} else {
-				flattened[strconv.Itoa(i)] = vt.String()
+				flattened[strconv.Itoa(i)] = typesv1.ValStr(vt.String())
 			}
 		case string:
-			flattened[strconv.Itoa(i)] = vt
+			flattened[strconv.Itoa(i)] = typesv1.ValStr(vt)
+		case bool:
+			flattened[strconv.Itoa(i)] = typesv1.ValBool(vt)
 		case []interface{}:
 			flattenedArrayFields := getFlattenedArrayFields(vt)
 			for k, v := range flattenedArrayFields {
@@ -153,7 +157,7 @@ func getFlattenedArrayFields(data []interface{}) map[string]string {
 				flattened[fmt.Sprintf("%d.%s", i, k)] = v
 			}
 		default:
-			flattened[strconv.Itoa(i)] = fmt.Sprintf("%v", vt)
+			flattened[strconv.Itoa(i)] = typesv1.ValStr(fmt.Sprintf("%v", vt))
 		}
 	}
 	return flattened
@@ -203,23 +207,26 @@ func (h *JSONHandler) UnmarshalJSON(data []byte) bool {
 	})
 
 	if h.Fields == nil {
-		h.Fields = make(map[string]string)
+		h.Fields = make(map[string]*typesv1.Val)
 	}
 
 	for key, val := range raw {
 		switch v := val.(type) {
 		case json.Number:
 			if z, err := v.Int64(); err == nil {
-				h.Fields[key] = fmt.Sprintf("%d", z)
+				h.Fields[key] = typesv1.ValI64(z)
 				continue
 			}
 			if f, err := v.Float64(); err == nil {
-				h.Fields[key] = fmt.Sprintf("%g", f)
+				h.Fields[key] = typesv1.ValF64(f)
 				continue
 			}
-			h.Fields[key] = v.String()
+			h.Fields[key] = typesv1.ValStr(v.String())
 		case string:
-			h.Fields[key] = fmt.Sprintf("%q", v)
+
+			h.Fields[key] = typesv1.ValStr(v)
+		case bool:
+			h.Fields[key] = typesv1.ValBool(v)
 		case []interface{}:
 			flattenedArrayFields := getFlattenedArrayFields(v)
 			for k, v := range flattenedArrayFields {
@@ -231,7 +238,7 @@ func (h *JSONHandler) UnmarshalJSON(data []byte) bool {
 				h.Fields[key+"."+keyNested] = val
 			}
 		default:
-			h.Fields[key] = fmt.Sprintf("%v", v)
+			h.Fields[key] = typesv1.ValStr(fmt.Sprintf("%v", v))
 		}
 	}
 

json_handler_test.go

@@ -1,4 +1,4 @@
-package humanlog_test
+package humanlog
 
 import (
 	"fmt"
@@ -7,7 +7,6 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	typesv1 "github.com/humanlogio/api/go/types/v1"
-	"github.com/humanlogio/humanlog"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/protobuf/testing/protocmp"
 	"google.golang.org/protobuf/types/known/timestamppb"
@@ -24,9 +23,9 @@ func TestJSONHandler_UnmarshalJSON_ParsesFields(t *testing.T) {
 
 	raw := []byte(fmt.Sprintf(`{ "message": %q, "level": %q, "time": %q }`, msg, level, tm))
 
-	opts := humanlog.DefaultOptions()
+	opts := DefaultOptions()
 
-	h := humanlog.JSONHandler{Opts: opts}
+	h := JSONHandler{Opts: opts}
 	ev := new(typesv1.StructuredLogEvent)
 	if !h.TryHandle(raw, ev) {
 		t.Fatalf("failed to parse log level")
@@ -56,12 +55,12 @@ func TestJSONHandler_UnmarshalJSON_ParsesCustomFields(t *testing.T) {
 
 	raw := []byte(fmt.Sprintf(`{ "mymessage": %q, "mylevel": %q, "mytime": %q }`, msg, level, tm))
 
-	opts := humanlog.DefaultOptions()
+	opts := DefaultOptions()
 	opts.LevelFields = []string{"mylevel"}
 	opts.MessageFields = []string{"mymessage"}
 	opts.TimeFields = []string{"mytime"}
 
-	h := humanlog.JSONHandler{Opts: opts}
+	h := JSONHandler{Opts: opts}
 
 	ev := new(typesv1.StructuredLogEvent)
 	if !h.TryHandle(raw, ev) {
@@ -91,12 +90,12 @@ func TestJSONHandler_UnmarshalJSON_ParsesCustomNestedFields(t *testing.T) {
 
 	raw := []byte(fmt.Sprintf(`{ "data": { "message": %q, "level": %q, "time": %q }}`, msg, level, tm))
 
-	opts := humanlog.DefaultOptions()
+	opts := DefaultOptions()
 	opts.LevelFields = []string{"data.level"}
 	opts.MessageFields = []string{"data.message"}
 	opts.TimeFields = []string{"data.time"}
 
-	h := humanlog.JSONHandler{Opts: opts}
+	h := JSONHandler{Opts: opts}
 	ev := new(typesv1.StructuredLogEvent)
 	if !h.TryHandle(raw, ev) {
 		t.Fatalf("failed to handle log")
@@ -134,12 +133,12 @@ func TestJSONHandler_UnmarshalJSON_ParsesCustomMultiNestedFields(t *testing.T) {
 	  }
 	}`, msg, level, tm))
 
-	opts := humanlog.DefaultOptions()
+	opts := DefaultOptions()
 	opts.LevelFields = []string{"data.l2.level"}
 	opts.MessageFields = []string{"data.l2.message"}
 	opts.TimeFields = []string{"data.l2.time"}
 
-	h := humanlog.JSONHandler{Opts: opts}
+	h := JSONHandler{Opts: opts}
 	ev := new(typesv1.StructuredLogEvent)
 	if !h.TryHandle(raw, ev) {
 		t.Fatalf("failed to handle log")
@@ -159,48 +158,48 @@ func TestJSONHandler_UnmarshalJSON_ParsesCustomMultiNestedFields(t *testing.T) {
 }
 
 func TestJsonHandler_TryHandle_LargeNumbers(t *testing.T) {
-	h := humanlog.JSONHandler{Opts: humanlog.DefaultOptions()}
+	h := JSONHandler{Opts: DefaultOptions()}
 	ev := new(typesv1.StructuredLogEvent)
 	raw := []byte(`{"storage":{"session.id":1730187806608637000, "some": {"float": 1.2345}}}`)
 	if !h.TryHandle(raw, ev) {
 		t.Fatalf("failed to handle log")
 	}
-	require.Equal(t, "1.2345", h.Fields["storage.some.float"])
-	require.Equal(t, "1730187806608637000", h.Fields["storage.session.id"])
+	require.Equal(t, 1.2345, h.Fields["storage.some.float"].GetF64())
+	require.Equal(t, int64(1730187806608637000), h.Fields["storage.session.id"].GetI64())
 }
 
 func TestJsonHandler_TryHandle_FlattendArrayFields(t *testing.T) {
-	handler := humanlog.JSONHandler{Opts: humanlog.DefaultOptions()}
+	handler := JSONHandler{Opts: DefaultOptions()}
 	ev := new(typesv1.StructuredLogEvent)
 	raw := []byte(`{"peers":[{"ID":"10.244.0.126:8083","URI":"10.244.0.126:8083"},{"ID":"10.244.0.206:8083","URI":"10.244.0.206:8083"},{"ID":"10.244.1.150:8083","URI":"10.244.1.150:8083"}],"storage":{"session.id":1730187806608637000, "some": {"float": 1.2345}}}`)
 	if !handler.TryHandle(raw, ev) {
 		t.Fatalf("failed to handle log")
 	}
-	require.Equal(t, "\"10.244.0.126:8083\"", handler.Fields["peers.0.ID"])
-	require.Equal(t, "\"10.244.0.126:8083\"", handler.Fields["peers.0.URI"])
-	require.Equal(t, "\"10.244.0.206:8083\"", handler.Fields["peers.1.ID"])
-	require.Equal(t, "\"10.244.0.206:8083\"", handler.Fields["peers.1.URI"])
-	require.Equal(t, "\"10.244.1.150:8083\"", handler.Fields["peers.2.ID"])
-	require.Equal(t, "\"10.244.1.150:8083\"", handler.Fields["peers.2.URI"])
+	require.Equal(t, "10.244.0.126:8083", handler.Fields["peers.0.ID"].GetStr())
+	require.Equal(t, "10.244.0.126:8083", handler.Fields["peers.0.URI"].GetStr())
+	require.Equal(t, "10.244.0.206:8083", handler.Fields["peers.1.ID"].GetStr())
+	require.Equal(t, "10.244.0.206:8083", handler.Fields["peers.1.URI"].GetStr())
+	require.Equal(t, "10.244.1.150:8083", handler.Fields["peers.2.ID"].GetStr())
+	require.Equal(t, "10.244.1.150:8083", handler.Fields["peers.2.URI"].GetStr())
 }
 
 func TestJsonHandler_TryHandle_FlattenedArrayFields_NestedArray(t *testing.T) {
-	handler := humanlog.JSONHandler{Opts: humanlog.DefaultOptions()}
+	handler := JSONHandler{Opts: DefaultOptions()}
 	ev := new(typesv1.StructuredLogEvent)
 	raw := []byte(`{"peers":[[1,2,3.14],[4,50.55,[6,7]],["hello","world"],{"foo":"bar"}]}`)
 	if !handler.TryHandle(raw, ev) {
 		t.Fatalf("failed to handle log")
 	}
-	require.Equal(t, "1", handler.Fields["peers.0.0"])
-	require.Equal(t, "2", handler.Fields["peers.0.1"])
-	require.Equal(t, "3.14", handler.Fields["peers.0.2"])
-	require.Equal(t, "4", handler.Fields["peers.1.0"])
-	require.Equal(t, "50.55", handler.Fields["peers.1.1"])
-	require.Equal(t, "6", handler.Fields["peers.1.2.0"])
-	require.Equal(t, "7", handler.Fields["peers.1.2.1"])
-	require.Equal(t, "hello", handler.Fields["peers.2.0"])
-	require.Equal(t, "world", handler.Fields["peers.2.1"])
-	require.Equal(t, "\"bar\"", handler.Fields["peers.3.foo"])
+	require.Equal(t, int64(1), handler.Fields["peers.0.0"].GetI64())
+	require.Equal(t, int64(2), handler.Fields["peers.0.1"].GetI64())
+	require.Equal(t, float64(3.14), handler.Fields["peers.0.2"].GetF64())
+	require.Equal(t, int64(4), handler.Fields["peers.1.0"].GetI64())
+	require.Equal(t, float64(50.55), handler.Fields["peers.1.1"].GetF64())
+	require.Equal(t, int64(6), handler.Fields["peers.1.2.0"].GetI64())
+	require.Equal(t, int64(7), handler.Fields["peers.1.2.1"].GetI64())
+	require.Equal(t, "hello", handler.Fields["peers.2.0"].GetStr())
+	require.Equal(t, "world", handler.Fields["peers.2.1"].GetStr())
+	require.Equal(t, "bar", handler.Fields["peers.3.foo"].GetStr())
 }
 
 func TestParseAsctimeFields(t *testing.T) {
@@ -223,8 +222,8 @@ func TestParseAsctimeFields(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			opts := humanlog.DefaultOptions()
-			h := humanlog.JSONHandler{Opts: opts}
+			opts := DefaultOptions()
+			h := JSONHandler{Opts: opts}
 			ev := new(typesv1.StructuredLogEvent)
 			if !h.TryHandle(test.raw, ev) {
 				t.Fatalf("failed to handle log")