Add tracing to Loki (grafana#328)

* Add tracing to loki

Signed-off-by: Goutham Veeramachaneni <[email protected]>

* Use the op names from common/server

We don't need to the tracing middleware again. Further, the op names are
not sanitised here.

Signed-off-by: Goutham Veeramachaneni <[email protected]>

Gopkg.toml

@@ -36,6 +36,14 @@
   name = "gopkg.in/fsnotify.v1"
   source = "https://github.com/fsnotify/fsnotify.git"
 
+[[override]]
+  name = "github.com/uber/jaeger-lib"
+  version = "2.0.0"
+
+[[override]]
+  name = "github.com/uber/jaeger-client-go"
+  branch = "master"
+
 # For Prometheus
 
 [[constraint]]
@@ -47,32 +55,12 @@
   name = "github.com/prometheus/common"
 
 [[override]]
-  name = "github.com/prometheus/tsdb"
-  revision = "6d489a1004dc"
-
-[[override]]
-  name = "k8s.io/api"
-  revision = "05914d821849"
-
-[[override]]
-  name = "k8s.io/apimachinery"
-  revision = "2b1284ed4c93"
-
-[[override]]
-  name = "k8s.io/client-go"
-  revision = "a47917edff34"
-
-[[override]]
-  name = "github.com/Azure/azure-sdk-for-go"
-  revision = "bd73d950fa44"
-
-[[override]]
-  name = "github.com/Azure/go-autorest"
-  version = "10.8.1"
+  branch = "master"
+  name = "github.com/prometheus/client_golang"
 
 [[override]]
-  version = "=v1.0.4"
-  name = "github.com/miekg/dns"
+  name = "github.com/prometheus/tsdb"
+  version = "0.6.1"
 
 [prune]
   go-tests = true

cmd/loki/main.go

@@ -2,13 +2,15 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"os"
 
 	"github.com/go-kit/kit/log/level"
 	"github.com/grafana/loki/pkg/helpers"
 	"github.com/grafana/loki/pkg/loki"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/version"
+	"github.com/weaveworks/common/tracing"
 
 	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/cortexproject/cortex/pkg/util/flagext"
@@ -36,6 +38,15 @@ func main() {
 		}
 	}
 
+	// Setting the environment variable JAEGER_AGENT_HOST enables tracing
+	trace := tracing.NewFromEnv(fmt.Sprintf("loki-%s", cfg.Target))
+	defer func() {
+		if err := trace.Close(); err != nil {
+			level.Error(util.Logger).Log("msg", "error closing tracing", "err", err)
+			os.Exit(1)
+		}
+	}()
+
 	t, err := loki.New(cfg)
 	if err != nil {
 		level.Error(util.Logger).Log("msg", "error initialising loki", "err", err)

pkg/loki/modules.go

@@ -5,8 +5,6 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/opentracing-contrib/go-stdlib/nethttp"
-	opentracing "github.com/opentracing/opentracing-go"
 	"google.golang.org/grpc/health/grpc_health_v1"
 
 	"github.com/cortexproject/cortex/pkg/chunk/storage"
@@ -114,13 +112,7 @@ func (t *Loki) initDistributor() (err error) {
 		return
 	}
 
-	operationNameFunc := nethttp.OperationNameFunc(func(r *http.Request) string {
-		return r.URL.RequestURI()
-	})
 	t.server.HTTP.Handle("/api/prom/push", middleware.Merge(
-		middleware.Func(func(handler http.Handler) http.Handler {
-			return nethttp.Middleware(opentracing.GlobalTracer(), handler, operationNameFunc)
-		}),
 		t.httpAuthMiddleware,
 	).Wrap(http.HandlerFunc(t.distributor.PushHandler)))
 
@@ -133,13 +125,7 @@ func (t *Loki) initQuerier() (err error) {
 		return
 	}
 
-	operationNameFunc := nethttp.OperationNameFunc(func(r *http.Request) string {
-		return r.URL.RequestURI()
-	})
 	httpMiddleware := middleware.Merge(
-		middleware.Func(func(handler http.Handler) http.Handler {
-			return nethttp.Middleware(opentracing.GlobalTracer(), handler, operationNameFunc)
-		}),
 		t.httpAuthMiddleware,
 	)
 	t.server.HTTP.Handle("/api/prom/query", httpMiddleware.Wrap(http.HandlerFunc(t.querier.QueryHandler)))