loki: use new runtimeconfig package from Cortex (grafana#1484)

* loki: use new runtimeconfig package from Cortex

runtimeconfig is a generalization of previous "overrides" mechanism,
and can be used not for per-tenant limits, but other parts of Loki
too (only one additional usage for now is "multi" KV client).

Signed-off-by: Peter Štibraný <[email protected]>

* go mod tidy, go mod vendor

Signed-off-by: Peter Štibraný <[email protected]>

* Removed RuntimeConfig from Ingester dependencies.

Ingester already depends on RuntimeConfig via Store and Overrides mechanism.

Signed-off-by: Peter Štibraný <[email protected]>

* Removed extra new line.

Signed-off-by: Peter Štibraný <[email protected]>

* Remove unused method.

Signed-off-by: Peter Štibraný <[email protected]>

* Fix broken code after rebasing.
Removed incorrect comment.

Signed-off-by: Peter Štibraný <[email protected]>

* Updated docs with section about runtime configuration. Mostly copied from Cortex.

Signed-off-by: Peter Štibraný <[email protected]>

* go mod tidy, go mod vendor

Signed-off-by: Peter Štibraný <[email protected]>

* Updated Cortex to current master.

Signed-off-by: Peter Štibraný <[email protected]>

* Go mod tidy, vendor

Signed-off-by: Peter Štibraný <[email protected]>

cmd/loki/main.go

@@ -39,10 +39,7 @@ func main() {
 	// This global is set to the config passed into the last call to `NewOverrides`. If we don't
 	// call it atleast once, the defaults are set to an empty struct.
 	// We call it with the flag values so that the config file unmarshalling only overrides the values set in the config.
-	if _, err := validation.NewOverrides(config.LimitsConfig); err != nil {
-		level.Error(util.Logger).Log("msg", "setting up overrides", "error", err)
-		os.Exit(1)
-	}
+	validation.SetDefaultLimitsForYAMLUnmarshalling(config.LimitsConfig)
 
 	// Init the logger which will honor the log level set in config.Server
 	if reflect.DeepEqual(&config.Server.LogLevel, &logging.Level{}) {

docs/configuration/README.md

@@ -24,6 +24,7 @@ Configuration examples can be found in the [Configuration Examples](examples.md)
 * [table_manager_config](#table_manager_config)
   * [provision_config](#provision_config)
     * [auto_scaling_config](#auto_scaling_config)
+* [Runtime Configuration file](#runtime-configuration-file)
 
 ## Configuration File Reference
 
@@ -88,6 +89,9 @@ Supported contents and default values of `loki.yaml`:
 
 # Configures the table manager for retention
 [table_manager: <table_manager_config>]
+
+# Configuration for "runtime config" module, responsible for reloading runtime configuration file.
+[runtime_config: <runtime_config>]
 ```
 
 ## server_config
@@ -799,10 +803,10 @@ logs in Loki.
 # Maximum number of stream matchers per query.
 [max_streams_matchers_per_query: <int> | default = 1000]
 
-# Filename of per-user overrides file
+# Feature renamed to 'runtime configuration', flag deprecated in favor of -runtime-config.file (runtime_config.file in YAML)
 [per_tenant_override_config: <string>]
 
-# Period with which to reload the overrides file if configured.
+# Feature renamed to 'runtime configuration', flag deprecated in favor of -runtime-config.reload-period (runtime_config.period in YAML)
 [per_tenant_override_period: <duration> | default = 10s]
 ```
 
@@ -906,3 +910,36 @@ The `auto_scaling_config` block configures autoscaling for DynamoDB.
 # DynamoDB target ratio of consumed capacity to provisioned capacity.
 [target: <float> | default = 80]
 ```
+
+## Runtime Configuration file
+
+Loki has a concept of "runtime config" file, which is simply a file that is reloaded while Loki is running. It is used by some Loki components to allow operator to change some aspects of Loki configuration without restarting it. File is specified by using `-runtime-config.file=<filename>` flag and reload period (which defaults to 10 seconds) can be changed by `-runtime-config.reload-period=<duration>` flag. Previously this mechanism was only used by limits overrides, and flags were called `-limits.per-user-override-config=<filename>` and `-limits.per-user-override-period=10s` respectively. These are still used, if `-runtime-config.file=<filename>` is not specified.
+
+At the moment, two components use runtime configuration: limits and multi KV store.
+
+Options for runtime configuration reload can also be configured via YAML:
+
+```yaml
+# Configuration file to periodically check and reload.
+[file: <string>: default = empty]
+
+# How often to check the file.
+[period: <duration>: default 10 seconds]
+```
+
+Example runtime configuration file:
+
+```yaml
+overrides:
+  tenant1:
+    ingestion_rate_mb: 10
+    max_streams_per_user: 100000
+    max_chunks_per_query: 100000
+  tenant2:
+    max_streams_per_user: 1000000
+    max_chunks_per_query: 1000000
+
+multi_kv_config:
+    mirror-enabled: false
+    primary: consul
+```

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/containerd/containerd v1.3.2 // indirect
 	github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448 // indirect
 	github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e
-	github.com/cortexproject/cortex v0.4.1-0.20191217132644-cd4009e2f8e7
+	github.com/cortexproject/cortex v0.4.1-0.20200122092731-ab3e8360fe30
 	github.com/davecgh/go-spew v1.1.1
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/docker v0.7.3-0.20190817195342-4760db040282
@@ -20,11 +20,11 @@ require (
 	github.com/fluent/fluent-bit-go v0.0.0-20190925192703-ea13c021720c
 	github.com/frankban/quicktest v1.7.2 // indirect
 	github.com/go-kit/kit v0.9.0
-	github.com/gogo/protobuf v1.3.0 // remember to update loki-build-image/Dockerfile too
+	github.com/gogo/protobuf v1.3.1 // remember to update loki-build-image/Dockerfile too
 	github.com/golang/snappy v0.0.1
 	github.com/gorilla/mux v1.7.1
 	github.com/gorilla/websocket v1.4.0
-	github.com/grpc-ecosystem/grpc-gateway v1.9.6 // indirect
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.1-0.20191002090509-6af20e3a5340 // indirect
 	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645
 	github.com/hashicorp/golang-lru v0.5.3
 	github.com/hpcloud/tail v1.0.0
@@ -38,29 +38,24 @@ require (
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/pierrec/lz4 v2.3.1-0.20191115212037-9085dacd1e1e+incompatible
 	github.com/pkg/errors v0.8.1
-	github.com/prometheus/client_golang v1.1.0
+	github.com/prometheus/client_golang v1.2.1
 	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4
 	github.com/prometheus/common v0.7.0
-	github.com/prometheus/prometheus v1.8.2-0.20190918104050-8744afdd1ea0
+	github.com/prometheus/prometheus v1.8.2-0.20191126064551-80ba03c67da1
 	github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749
 	github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd
 	github.com/stretchr/testify v1.4.0
 	github.com/tonistiigi/fifo v0.0.0-20190226154929-a9fb20d87448
 	github.com/ugorji/go v1.1.7 // indirect
 	github.com/weaveworks/common v0.0.0-20191103151037-0e7cefadc44f
 	go.etcd.io/etcd v0.0.0-20190815204525-8f85f0dc2607 // indirect
-	go.opencensus.io v0.22.1 // indirect
-	golang.org/x/net v0.0.0-20190923162816-aa69164e4478
-	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
+	golang.org/x/net v0.0.0-20191112182307-2180aed22343
 	golang.org/x/sys v0.0.0-20191218084908-4a24b4065292 // indirect
-	golang.org/x/tools v0.0.0-20190925134113-a044388aa56f // indirect
-	google.golang.org/appengine v1.6.3 // indirect
-	google.golang.org/genproto v0.0.0-20190916214212-f660b8655731 // indirect
 	google.golang.org/grpc v1.25.1
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/fsnotify.v1 v1.4.7
-	gopkg.in/yaml.v2 v2.2.2
-	k8s.io/klog v0.4.0
+	gopkg.in/yaml.v2 v2.2.5
+	k8s.io/klog v1.0.0
 )
 
 replace github.com/hpcloud/tail => github.com/grafana/tail v0.0.0-20191024143944-0b54ddf21fe7