promtail: Add systemd journal support (grafana#730)

Support for reading systemd journal entries has been added. 
promtail will look for a job in scrape_configs with a journal key 
to activate the journal target. 

If GOOS=linux and CGO_ENABLED=1, promtail will now require 
libsystemd headers to be available for building. If GOOS is not 
linux or CGO_ENABLED is not 1, journal support will be unavailable
and a log message will be printed warning the user that their config 
file has journal tailing configured without it being built into promtail. 

See docs/promtail-examples.md for a concrete example of 
using journal support. 

Other structural changes made: 

  1. Ability for checking if scrape.Config.ServiceDiscoveryConfig is 
     non-zero has been added. 

     This was chosen over making ServiceDiscoveryConfig a pointer
     as yaml.v2 cannot parse an inline struct into a pointer value. 

  2. Updated pipeline logger component name to journal_pipeline and
     file_pipeline for JournalTargetManager and FileTargetManager
     respectively.

  3. The positions file will now store positions as strings instead of 
     integers. Existing positions will be read in properly but written out 
     as strings the next time the positions file is saved. This is done to 
     be able to store the journal cursor, which is a string. The positions 
     API has been updated to support reading in the old integer values 
     and the new string values.

.circleci/config.yml

@@ -45,7 +45,7 @@ workflows:
 # https://circleci.com/blog/circleci-hacks-reuse-yaml-in-your-circleci-config-with-yaml/
 defaults: &defaults
   docker:
-    - image: grafana/loki-build-image:0.2.1
+    - image: grafana/loki-build-image:0.3.0
   working_directory: /go/src/github.com/grafana/loki
 
 jobs:
@@ -156,7 +156,7 @@ jobs:
           key: v1-loki-{{ .Branch }}-{{ .Revision }}
       - restore_cache:
           key: v1-loki-plugin-{{ .Branch }}-{{ .Revision }}
-          
+
       - run:
           name: Load Images
           command: |
@@ -168,7 +168,7 @@ jobs:
           command: |
             docker login -u "$DOCKER_USER" -p "$DOCKER_PASS" &&
             make push-latest
-      
+
       - run:
           name: Push Docker Plugin
           command: |

Gopkg.toml

@@ -74,4 +74,4 @@
 
 [[override]]
   name = "k8s.io/client-go"
-  revision = "1a26190bd76a9017e289958b9fba936430aa3704"
+  revision = "1a26190bd76a9017e289958b9fba936430aa3704"

Makefile

@@ -104,6 +104,7 @@ loki-build-image/$(UPTODATE): loki-build-image/*
 # All the boiler plate for building golang follows:
 SUDO := $(shell docker info >/dev/null 2>&1 || echo "sudo -E")
 BUILD_IN_CONTAINER := true
+CGO_ENABLED := 0
 # RM is parameterized to allow CircleCI to run builds, as it
 # currently disallows `docker run --rm`. This value is overridden
 # in circle.yml
@@ -149,13 +150,13 @@ $(EXES) $(DEBUG_EXES) $(PROTO_GOS) $(YACC_GOS) lint test shell check-generated-f
 else
 
 $(DEBUG_EXES): loki-build-image/$(UPTODATE)
-	CGO_ENABLED=0 go build $(DEBUG_GO_FLAGS) -o $@ ./$(@D)
+	CGO_ENABLED=$(CGO_ENABLED) go build $(DEBUG_GO_FLAGS) -o $@ ./$(@D)
 	$(NETGO_CHECK)
 	# Copy the delve binary to make it easily available to put in the binary's container.
 	[ -f "/go/bin/dlv" ] && mv "/go/bin/dlv" $(@D)/dlv
 
 $(EXES): loki-build-image/$(UPTODATE)
-	CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D)
+	CGO_ENABLED=$(CGO_ENABLED) go build $(GO_FLAGS) -o $@ ./$(@D)
 	$(NETGO_CHECK)
 
 %.pb.go: loki-build-image/$(UPTODATE)

build/Dockerfile

@@ -12,12 +12,12 @@ ARG GOARCH="amd64"
 COPY . /go/src/github.com/grafana/loki
 WORKDIR /go/src/github.com/grafana/loki
 RUN touch loki-build-image/.uptodate &&\
-    mkdir /build
+  mkdir /build
 
 # production image
 FROM golang as builder-production
 ARG APP
-RUN make BUILD_IN_CONTAINER=false cmd/${APP}/${APP} &&\
+RUN make CGO_ENABLED=1 BUILD_IN_CONTAINER=false cmd/${APP}/${APP} &&\
   mv cmd/${APP}/${APP} /build/${APP}
 
 FROM scratch as production
@@ -28,7 +28,7 @@ COPY --from=builder-production /build/${APP} /usr/bin/${APP}
 FROM golang as builder-debug
 ARG APP
 RUN go get github.com/go-delve/delve/cmd/dlv &&\
-  make BUILD_IN_CONTAINER=false cmd/promtail/promtail-debug &&\
+  make CGO_ENBALED=1 BUILD_IN_CONTAINER=false cmd/promtail/promtail-debug &&\
   mv cmd/${APP}/${APP}-debug /build/app-debug &&\
   mv cmd/${APP}/dlv /build/dlv
 

docs/promtail-examples.md

@@ -10,7 +10,7 @@ This example of config promtail based on original docker [config](https://github
 and show how work with 2 and more sources:
 
 Filename for example: my-docker-config.yaml
-```
+```yaml
 server:
   http_listen_port: 9080
   grpc_listen_port: 0
@@ -45,6 +45,7 @@ scrape_configs:
       __path__: /srv/log/someone_service/*.log
 
 ```
+
 #### Description
 
 Scrape_config section of config.yaml contents contains various jobs for parsing your logs
@@ -54,15 +55,79 @@ Scrape_config section of config.yaml contents contains various jobs for parsing
 `__path__` it is path to directory where stored your logs.
 
 If you run promtail and this config.yaml in Docker container, don't forget use docker volumes for mapping real directories
-with log to those folders in the container. 
+with log to those folders in the container.
 
 #### Example Use
 1) Create folder, for example `promtail`, then new sub directory `build/conf` and place there `my-docker-config.yaml`.
 2) Create new Dockerfile in root folder `promtail`, with contents
-```
+```dockerfile
 FROM grafana/promtail:latest
 COPY build/conf /etc/promtail
 ```
 3) Create your Docker image based on original Promtail image and tag it, for example `mypromtail-image`
 3) After that you can run Docker container by this command:
 `docker run -d --name promtail --network loki_network -p 9080:9080 -v /var/log:/var/log -v /srv/log/someone_service:/srv/log/someone_service mypromtail-image -config.file=/etc/promtail/my-docker-config.yaml`
+
+## Simple Systemd Journal Config
+
+This example demonstrates how to configure promtail to listen to systemd journal
+entries and write them to Loki:
+
+Filename for example: my-systemd-journal-config.yaml
+
+```yaml
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://ip_or_hostname_where_loki_runns:3100/api/prom/push
+
+scrape_configs:
+  - job_name: journal
+    journal:
+      path: /var/log/journal
+      labels:
+        job: systemd-journal
+    relabel_configs:
+      - source_labels: ['__journal__systemd_unit']
+        target_label: 'unit'
+```
+
+### Description
+
+Just like the Docker example, the `scrape_configs` sections holds various
+jobs for parsing logs. A job with a `journal` key configures it for systemd
+journal reading.
+
+`path` is an optional string specifying the path to read journal entries
+from. If unspecified, defaults to the system default (`/var/log/journal`).
+
+`labels`: is a map of string values specifying labels that should always
+be associated with each log entry being read from the systemd journal.
+In our example, each log will have a label of `job=systemd-journal`.
+
+Every field written to the systemd journal is available for processing
+in the `relabel_configs` section. Label names are converted to lowercase
+and prefixed with `__journal_`. After `relabel_configs` processes all
+labels for a job entry, any label starting with `__` is deleted.
+
+Our example renames the `_SYSTEMD_UNIT` label (available as
+`__journal__systemd_unit` in promtail) to `unit` so it will be available
+in Loki. All other labels from the journal entry are dropped.
+
+### Example Use
+
+`promtail` must have access to the journal path (`/var/log/journal`)
+where journal entries are stored for journal support to work correctly.
+
+If running with Docker, that means to bind that path:
+
+```bash
+docker run -d --name promtail --network loki_network -p 9080:9080 \
+  -v /var/log/journal:/var/log/journal \
+  mypromtail-image -config.file=/etc/promtail/my-systemd-journal-config.yaml
+```

loki-build-image/Dockerfile

@@ -1,23 +1,23 @@
 FROM golang:1.11.4-stretch
-RUN apt-get update && apt-get install -y file jq unzip protobuf-compiler libprotobuf-dev && \
-    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+RUN apt-get update && apt-get install -y file jq unzip protobuf-compiler libprotobuf-dev libsystemd-dev && \
+	rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 ENV DOCKER_VER="17.03.0-ce"
 RUN curl -L -o /tmp/docker-$DOCKER_VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$DOCKER_VER.tgz && \
-    tar -xz -C /tmp -f /tmp/docker-$DOCKER_VER.tgz && \
-    mv /tmp/docker/* /usr/bin && \
-		rm /tmp/docker-$DOCKER_VER.tgz
+	tar -xz -C /tmp -f /tmp/docker-$DOCKER_VER.tgz && \
+	mv /tmp/docker/* /usr/bin && \
+	rm /tmp/docker-$DOCKER_VER.tgz
 ENV HELM_VER="v2.13.1"
 RUN curl -L -o /tmp/helm-$HELM_VER.tgz http://storage.googleapis.com/kubernetes-helm/helm-${HELM_VER}-linux-amd64.tar.gz && \
-    tar -xz -C /tmp -f /tmp/helm-$HELM_VER.tgz && \
-    mv /tmp/linux-amd64/helm /usr/bin/helm && \
-    rm -rf /tmp/linux-amd64 /tmp/helm-$HELM_VER.tgz
+	tar -xz -C /tmp -f /tmp/helm-$HELM_VER.tgz && \
+	mv /tmp/linux-amd64/helm /usr/bin/helm && \
+	rm -rf /tmp/linux-amd64 /tmp/helm-$HELM_VER.tgz
 RUN go get \
-		github.com/golang/protobuf/protoc-gen-go \
-		github.com/gogo/protobuf/protoc-gen-gogoslick \
-		github.com/gogo/protobuf/gogoproto \
-		github.com/go-delve/delve/cmd/dlv \
-		golang.org/x/tools/cmd/goyacc && \
-		rm -rf /go/pkg /go/src
+	github.com/golang/protobuf/protoc-gen-go \
+	github.com/gogo/protobuf/protoc-gen-gogoslick \
+	github.com/gogo/protobuf/gogoproto \
+	github.com/go-delve/delve/cmd/dlv \
+	golang.org/x/tools/cmd/goyacc && \
+	rm -rf /go/pkg /go/src
 ENV GOLANGCI_LINT_COMMIT="692dacb773b703162c091c2d8c59f9cd2d6801db"
 RUN mkdir -p $(go env GOPATH)/src/github.com/golangci/ && git clone https://github.com/golangci/golangci-lint.git $(go env GOPATH)/src/github.com/golangci/golangci-lint && \
 	cd $(go env GOPATH)/src/github.com/golangci/golangci-lint  && git checkout ${GOLANGCI_LINT_COMMIT} && cd cmd/golangci-lint/  &&\

pkg/promtail/positions/positions.go

@@ -5,6 +5,8 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -32,14 +34,14 @@ type Positions struct {
 	logger    log.Logger
 	cfg       Config
 	mtx       sync.Mutex
-	positions map[string]int64
+	positions map[string]string
 	quit      chan struct{}
 	done      chan struct{}
 }
 
 // File format for the positions data.
 type File struct {
-	Positions map[string]int64 `yaml:"positions"`
+	Positions map[string]string `yaml:"positions"`
 }
 
 // New makes a new Positions.
@@ -67,20 +69,42 @@ func (p *Positions) Stop() {
 	<-p.done
 }
 
-// Put records (asynchronously) how far we've read through a file.
-func (p *Positions) Put(path string, pos int64) {
+// PutString records (asynchronsouly) how far we've read through a file.
+// Unlike Put, it records a string offset and is only useful for
+// JournalTargets which doesn't have integer offsets.
+func (p *Positions) PutString(path string, pos string) {
 	p.mtx.Lock()
 	defer p.mtx.Unlock()
 	p.positions[path] = pos
 }
 
-// Get returns how far we've read through a file.
-func (p *Positions) Get(path string) int64 {
+// Put records (asynchronously) how far we've read through a file.
+func (p *Positions) Put(path string, pos int64) {
+	p.PutString(path, strconv.FormatInt(pos, 10))
+}
+
+// GetString returns how far we've through a file as a string.
+// JournalTarget writes a journal cursor to the positions file, while
+// FileTarget writes an integer offset. Use Get to read the integer
+// offset.
+func (p *Positions) GetString(path string) string {
 	p.mtx.Lock()
 	defer p.mtx.Unlock()
 	return p.positions[path]
 }
 
+// Get returns how far we've read through a file. Returns an error
+// if the value stored for the file is not an integer.
+func (p *Positions) Get(path string) (int64, error) {
+	p.mtx.Lock()
+	defer p.mtx.Unlock()
+	pos, ok := p.positions[path]
+	if !ok {
+		return 0, nil
+	}
+	return strconv.ParseInt(pos, 10, 64)
+}
+
 // Remove removes the position tracking for a filepath
 func (p *Positions) Remove(path string) {
 	p.mtx.Lock()
@@ -118,7 +142,7 @@ func (p *Positions) run() {
 
 func (p *Positions) save() {
 	p.mtx.Lock()
-	positions := make(map[string]int64, len(p.positions))
+	positions := make(map[string]string, len(p.positions))
 	for k, v := range p.positions {
 		positions[k] = v
 	}
@@ -134,6 +158,12 @@ func (p *Positions) cleanup() {
 	defer p.mtx.Unlock()
 	toRemove := []string{}
 	for k := range p.positions {
+		// If the position file is prefixed with journal, it's a
+		// JournalTarget cursor and not a file on disk.
+		if strings.HasPrefix(k, "journal-") {
+			continue
+		}
+
 		if _, err := os.Stat(k); err != nil {
 			if os.IsNotExist(err) {
 				// File no longer exists.
@@ -150,11 +180,11 @@ func (p *Positions) cleanup() {
 	}
 }
 
-func readPositionsFile(filename string) (map[string]int64, error) {
+func readPositionsFile(filename string) (map[string]string, error) {
 	buf, err := ioutil.ReadFile(filepath.Clean(filename))
 	if err != nil {
 		if os.IsNotExist(err) {
-			return map[string]int64{}, nil
+			return map[string]string{}, nil
 		}
 		return nil, err
 	}
@@ -167,7 +197,7 @@ func readPositionsFile(filename string) (map[string]int64, error) {
 	return p.Positions, nil
 }
 
-func writePositionFile(filename string, positions map[string]int64) error {
+func writePositionFile(filename string, positions map[string]string) error {
 	buf, err := yaml.Marshal(File{
 		Positions: positions,
 	})