diff --git a/.circleci/config.yml b/.circleci/config.yml index 8842d3449cda1..58a5028f7e105 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,7 +47,7 @@ workflows: # https://circleci.com/blog/circleci-hacks-reuse-yaml-in-your-circleci-config-with-yaml/ .defaults: &defaults docker: - - image: grafana/loki-build-image:0.9.2 + - image: grafana/loki-build-image:0.9.3 working_directory: /src/loki jobs: diff --git a/.drone/drone.yml b/.drone/drone.yml index 1030be908f683..e5e2f01c31d00 100644 --- a/.drone/drone.yml +++ b/.drone/drone.yml @@ -12,28 +12,28 @@ workspace: steps: - name: test - image: grafana/loki-build-image:0.9.2 + image: grafana/loki-build-image:0.9.3 commands: - make BUILD_IN_CONTAINER=false test depends_on: - clone - name: lint - image: grafana/loki-build-image:0.9.2 + image: grafana/loki-build-image:0.9.3 commands: - make BUILD_IN_CONTAINER=false lint depends_on: - clone - name: check-generated-files - image: grafana/loki-build-image:0.9.2 + image: grafana/loki-build-image:0.9.3 commands: - make BUILD_IN_CONTAINER=false check-generated-files depends_on: - clone - name: check-mod - image: grafana/loki-build-image:0.9.2 + image: grafana/loki-build-image:0.9.3 commands: - make BUILD_IN_CONTAINER=false check-mod depends_on: @@ -638,7 +638,7 @@ platform: steps: - name: trigger - image: grafana/loki-build-image:0.9.2 + image: grafana/loki-build-image:0.9.3 commands: - ./tools/deploy.sh environment: @@ -666,7 +666,7 @@ platform: steps: - name: trigger - image: grafana/loki-build-image:0.9.2 + image: grafana/loki-build-image:0.9.3 commands: - go run ./tools/delete_tags.go -max-age=2160h -repo grafana/loki -delete - go run ./tools/delete_tags.go -max-age=2160h -repo grafana/promtail -delete diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b2390dde565fb..470017acff1b9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,7 +26,7 @@ Please follow the [Loki Helm Chart](./production/helm/README.md). ### Dependency management We use [Go modules](https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more) to manage dependencies on external packages. -This requires a working Go environment with version 1.13 or greater and git installed. +This requires a working Go environment with version 1.14 or greater and git installed. To add or update a new dependency, use the `go get` command: @@ -67,4 +67,3 @@ import ( "github.com/grafana/loki/pkg/logql" ) ``` - diff --git a/Makefile b/Makefile index 24e6a95542c03..c2f8205a8725f 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ IMAGE_NAMES := $(foreach dir,$(DOCKER_IMAGE_DIRS),$(patsubst %,$(IMAGE_PREFIX)%, # make BUILD_IN_CONTAINER=false target # or you can override this with an environment variable BUILD_IN_CONTAINER ?= true -BUILD_IMAGE_VERSION := 0.9.2 +BUILD_IMAGE_VERSION := 0.9.3 # Docker image info IMAGE_PREFIX ?= grafana diff --git a/cmd/chunks-inspect/go.mod b/cmd/chunks-inspect/go.mod index 7ffd87fbcb697..91054cf5f97fe 100644 --- a/cmd/chunks-inspect/go.mod +++ b/cmd/chunks-inspect/go.mod @@ -1,6 +1,6 @@ module github.com/grafana/loki/cmd/chunks-inspect -go 1.13 +go 1.14 require ( github.com/frankban/quicktest v1.7.2 // indirect diff --git a/cmd/fluent-bit/Dockerfile b/cmd/fluent-bit/Dockerfile index aff1fdb812e41..04fbfdd310c2c 100644 --- a/cmd/fluent-bit/Dockerfile +++ b/cmd/fluent-bit/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.13 as build +FROM golang:1.14.2 as build COPY . /src/loki WORKDIR /src/loki RUN make clean && make BUILD_IN_CONTAINER=false fluent-bit-plugin diff --git a/cmd/loki-canary/Dockerfile b/cmd/loki-canary/Dockerfile index 6a9d8f41155ea..dc32f8b4cc303 100644 --- a/cmd/loki-canary/Dockerfile +++ b/cmd/loki-canary/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.13 as build +FROM golang:1.14.2 as build # TOUCH_PROTOS signifies if we should touch the compiled proto files and thus not regenerate them. # This is helpful when file system timestamps can't be trusted with make ARG TOUCH_PROTOS diff --git a/cmd/loki/Dockerfile b/cmd/loki/Dockerfile index 82ad9f6a93b52..82d295250ad7f 100644 --- a/cmd/loki/Dockerfile +++ b/cmd/loki/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.13 as build +FROM golang:1.14.2 as build # TOUCH_PROTOS signifies if we should touch the compiled proto files and thus not regenerate them. # This is helpful when file system timestamps can't be trusted with make ARG TOUCH_PROTOS diff --git a/cmd/promtail/Dockerfile b/cmd/promtail/Dockerfile index 78754b6e3c116..c74e75bc06dd0 100644 --- a/cmd/promtail/Dockerfile +++ b/cmd/promtail/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.13 as build +FROM golang:1.14.2 as build # TOUCH_PROTOS signifies if we should touch the compiled proto files and thus not regenerate them. # This is helpful when file system timestamps can't be trusted with make ARG TOUCH_PROTOS diff --git a/cmd/promtail/Dockerfile.arm32 b/cmd/promtail/Dockerfile.arm32 index 37f9bfc11d63e..891aa6550f28b 100644 --- a/cmd/promtail/Dockerfile.arm32 +++ b/cmd/promtail/Dockerfile.arm32 @@ -1,4 +1,4 @@ -FROM golang:1.13 as build +FROM golang:1.14.2 as build # TOUCH_PROTOS signifies if we should touch the compiled proto files and thus not regenerate them. # This is helpful when file system timestamps can't be trusted with make ARG TOUCH_PROTOS diff --git a/docs/installation/install-from-source.md b/docs/installation/install-from-source.md index 53584369705e4..52d9b14e1ab25 100644 --- a/docs/installation/install-from-source.md +++ b/docs/installation/install-from-source.md @@ -4,7 +4,7 @@ In order to build Loki manually, you need to clone the GitHub repo and then `mak ## Prerequisites -- Go 1.13 or later +- Go 1.14 or later - Make - Docker (for updating protobuf files and yacc files) diff --git a/docs/installation/local.md b/docs/installation/local.md index 2a65b5a0716e0..a51617276d4d7 100644 --- a/docs/installation/local.md +++ b/docs/installation/local.md @@ -1,12 +1,12 @@ # Install and run Loki locally -In order to log events with Loki, you must download and install both Promtail and Loki. +In order to log events with Loki, you must download and install both Promtail and Loki. - Loki is the logging engine. - Promtail sends logs to Loki. ## Install and run -1. Navigate to the [release page](https://github.com/grafana/loki/releases/). +1. Navigate to the [release page](https://github.com/grafana/loki/releases/). 2. Scroll down to the Assets section under the version that you want to install. 3. Download the Loki and Promtail .zip files that correspond to your system. **Note:** Do not download LogCLI or Loki Canary at this time. [LogCLI](./getting-started/logcli.md) allows you to run Loki queries in a command line interface. [Loki Canary](./operations/loki-canary.md) is a tool to audit Loki performance. diff --git a/go.mod b/go.mod index fd776fd648ebb..40f12e2506d97 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/grafana/loki -go 1.13 +go 1.14 require ( github.com/Microsoft/go-winio v0.4.12 // indirect @@ -10,7 +10,7 @@ require ( github.com/containerd/containerd v1.3.2 // indirect github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf - github.com/cortexproject/cortex v1.1.1-0.20200616130854-34b45d1180c3 + github.com/cortexproject/cortex v1.1.1-0.20200625134921-0ea2318b0174 github.com/davecgh/go-spew v1.1.1 github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v0.7.3-0.20190817195342-4760db040282 @@ -46,7 +46,7 @@ require ( github.com/prometheus/client_golang v1.6.1-0.20200604110148-03575cad4e55 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.10.0 - github.com/prometheus/prometheus v1.8.2-0.20200609052543-1627d234da06 + github.com/prometheus/prometheus v1.8.2-0.20200622142935-153f859b7499 github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd github.com/stretchr/testify v1.5.1 @@ -54,7 +54,7 @@ require ( github.com/uber/jaeger-client-go v2.23.1+incompatible github.com/ugorji/go v1.1.7 // indirect github.com/weaveworks/common v0.0.0-20200512154658-384f10054ec5 - go.etcd.io/bbolt v1.3.4 + go.etcd.io/bbolt v1.3.5-0.20200615073812-232d8fc87f50 golang.org/x/net v0.0.0-20200602114024-627f9648deb9 google.golang.org/grpc v1.29.1 gopkg.in/alecthomas/kingpin.v2 v2.2.6 diff --git a/go.sum b/go.sum index 350dd0c2920a9..48ab85eb11f68 100644 --- a/go.sum +++ b/go.sum @@ -59,8 +59,6 @@ github.com/Azure/go-autorest v13.3.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.9.3-0.20191028180845-3492b2aff503 h1:uUhdsDMg2GbFLF5GfQPtLMWd5vdDZSfqvqQp3waafxQ= github.com/Azure/go-autorest/autorest v0.9.3-0.20191028180845-3492b2aff503/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest v0.10.0 h1:mvdtztBqcL8se7MdrUweNieTNi4kfNG6GOJuurQJpuY= -github.com/Azure/go-autorest/autorest v0.10.0/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= github.com/Azure/go-autorest/autorest v0.10.2 h1:NuSF3gXetiHyUbVdneJMEVyPUYAe5wh+aN08JYAf1tI= github.com/Azure/go-autorest/autorest v0.10.2/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= @@ -158,9 +156,7 @@ github.com/aws/aws-sdk-go v1.17.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aws/aws-sdk-go v1.22.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.25.48 h1:J82DYDGZHOKHdhx6hD24Tm30c2C3GchYGfN0mf9iKUk= github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.27.0 h1:0xphMHGMLBrPMfxR2AmVjZKcMEESEgWF8Kru94BNByk= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.30.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.31.9 h1:n+b34ydVfgC30j0Qm69yaapmjejQPW2BoDBX7Uy/tLI= github.com/aws/aws-sdk-go v1.31.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= @@ -202,6 +198,9 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chromedp/cdproto v0.0.0-20200116234248-4da64dd111ac/go.mod h1:PfAWWKJqjlGFYJEidUM6aVIWPr0EpobeyVWEEmplX7g= +github.com/chromedp/cdproto v0.0.0-20200424080200-0de008e41fa0/go.mod h1:PfAWWKJqjlGFYJEidUM6aVIWPr0EpobeyVWEEmplX7g= +github.com/chromedp/chromedp v0.5.3/go.mod h1:YLdPtndaHQ4rCpSpBG+IPpy9JvX0VD+7aaLxYgYj28w= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -233,8 +232,8 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cortexproject/cortex v0.6.1-0.20200228110116-92ab6cbe0995/go.mod h1:3Xa3DjJxtpXqxcMGdk850lcIRb81M0fyY1MQ6udY134= -github.com/cortexproject/cortex v1.1.1-0.20200616130854-34b45d1180c3 h1:PebLohuxr0MSgWYAc/iUCFpxIdZyISk7jKUuQKTRbRw= -github.com/cortexproject/cortex v1.1.1-0.20200616130854-34b45d1180c3/go.mod h1:gySKqWpKaHBAFML0dOQyglOhxlJ/Eyfslf4QNBJjPoY= +github.com/cortexproject/cortex v1.1.1-0.20200625134921-0ea2318b0174 h1:b9GU5db5TuOz9pry6Ed4Nyu/sWuxGjDe1dQrzJtUzlU= +github.com/cortexproject/cortex v1.1.1-0.20200625134921-0ea2318b0174/go.mod h1:oE48OnEVmpMi0prqknK/DnNCqIBAXZ6NIebKB5KHtYE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= @@ -259,6 +258,8 @@ github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMa github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dhui/dktest v0.3.0 h1:kwX5a7EkLcjo7VpsPQSYJcKGbXBXdjI9FGjuUj1jn6I= github.com/dhui/dktest v0.3.0/go.mod h1:cyzIUfGsBEbZ6BT7tnXqAShHSXCZhSNmFl70sZ7c1yc= +github.com/digitalocean/godo v1.37.0 h1:NEj5ne2cvLBHo1GJY1DNN/iEt9ipa72CMwwAjKEA530= +github.com/digitalocean/godo v1.37.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU= github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= @@ -462,6 +463,9 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/godbus/dbus v0.0.0-20190402143921-271e53dc4968/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/gofrs/flock v0.7.1 h1:DP+LD/t0njgoPBvT5MJLeliUIVQR03hiKR6vezdwHlc= github.com/gofrs/flock v0.7.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= @@ -504,14 +508,12 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= @@ -548,7 +550,6 @@ github.com/google/pprof v0.0.0-20190723021845-34ac40c74b70/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200417002340-c6e0a841f49a/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200507031123-427632fa3b1c/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -573,8 +574,6 @@ github.com/gophercloud/gophercloud v0.3.0 h1:6sjpKIpVwRIIwmcEGp+WwNovNsem+c+2vm6 github.com/gophercloud/gophercloud v0.3.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gophercloud/gophercloud v0.6.0 h1:Xb2lcqZtml1XjgYZxbeayEemq7ASbeTp09m36gQFpEU= github.com/gophercloud/gophercloud v0.6.0/go.mod h1:GICNByuaEBibcjmjvI7QvYJSZEbGkcYwAR7EZK2WMqM= -github.com/gophercloud/gophercloud v0.10.0 h1:Et+UGxoD72pK6K+46uOwyVxbtXJ6KBkWAegXBmqlf6c= -github.com/gophercloud/gophercloud v0.10.0/go.mod h1:gmC5oQqMDOMO1t1gq5DquX/yAU808e/4mzjjDA76+Ss= github.com/gophercloud/gophercloud v0.11.0 h1:pYMP9UZBdQa3lsfIZ1tZor4EbtxiuB6BHhocenkiH/E= github.com/gophercloud/gophercloud v0.11.0/go.mod h1:gmC5oQqMDOMO1t1gq5DquX/yAU808e/4mzjjDA76+Ss= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -609,8 +608,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.4/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.12.1 h1:zCy2xE9ablevUOrUZc3Dl72Dt+ya2FNAvC2yLYMHzi4= github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= -github.com/grpc-ecosystem/grpc-gateway v1.14.4 h1:IOPK2xMPP3aV6/NPt4jt//ELFo3Vv8sDVD8j3+tleDU= -github.com/grpc-ecosystem/grpc-gateway v1.14.4/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0= github.com/grpc-ecosystem/grpc-gateway v1.14.6 h1:8ERzHx8aj1Sc47mu9n/AksaKCSWrMchFtkdrS4BIj5o= github.com/grpc-ecosystem/grpc-gateway v1.14.6/go.mod h1:zdiPV4Yse/1gnckTHtghG4GkDEdKCRJduHpTxT3/jcw= github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU= @@ -759,6 +756,7 @@ github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08/go.mod h1:dFWs1zEqDjFtnBXsd1vPOZaLsESovai349994nHx3e0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -992,7 +990,6 @@ github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNk github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_golang v1.6.0 h1:YVPodQOcK15POxhgARIvnDRVpLcuK8mglnMrWfyrw6A= github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4= github.com/prometheus/client_golang v1.6.1-0.20200604110148-03575cad4e55 h1:ADHic9K/n5JQDFx/OAQbxkwR+uPJ9oYmN0taBMyYrBo= github.com/prometheus/client_golang v1.6.1-0.20200604110148-03575cad4e55/go.mod h1:25h+Uz1WvXDBZYwqGX8PAb71RBkcjxEVV/R5wGnsq4I= @@ -1039,9 +1036,9 @@ github.com/prometheus/prometheus v0.0.0-20190818123050-43acd0e2e93f/go.mod h1:rM github.com/prometheus/prometheus v1.8.2-0.20200107122003-4708915ac6ef/go.mod h1:7U90zPoLkWjEIQcy/rweQla82OCTUzxVHE51G3OhJbI= github.com/prometheus/prometheus v1.8.2-0.20200213233353-b90be6f32a33 h1:HBYrMJj5iosUjUkAK9L5GO+5eEQXbcrzdjkqY9HV5W4= github.com/prometheus/prometheus v1.8.2-0.20200213233353-b90be6f32a33/go.mod h1:fkIPPkuZnkXyopYHmXPxf9rgiPkVgZCN8w9o8+UgBlY= -github.com/prometheus/prometheus v1.8.2-0.20200601152113-3268eac2ddda/go.mod h1:QKHYbx8sTY1fj75M+lL+LhzDSFM00+dOBlFn5wBi+14= -github.com/prometheus/prometheus v1.8.2-0.20200609052543-1627d234da06 h1:VeGF2wHPK5Y2zvj+lqvizJtfRsEkC22YT5mT8Wl1vbk= -github.com/prometheus/prometheus v1.8.2-0.20200609052543-1627d234da06/go.mod h1:CwaXafRa0mm72de2GQWtfQxjGytbSKIGivWxQvjpRZs= +github.com/prometheus/prometheus v1.8.2-0.20200609165731-66dfb951c4ca/go.mod h1:CwaXafRa0mm72de2GQWtfQxjGytbSKIGivWxQvjpRZs= +github.com/prometheus/prometheus v1.8.2-0.20200622142935-153f859b7499 h1:q+yGm39CmSV1S7oxCz36nlvx9ugRoEodwuHusgJw+iU= +github.com/prometheus/prometheus v1.8.2-0.20200622142935-153f859b7499/go.mod h1:QV6T0PPQi5UFmqcLBJw3JiyIR8r1O7KEv9qlVw4VV40= github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1 h1:+kGqA4dNN5hn7WwvKdzHl0rdN5AEkbNZd0VjRltAiZg= github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1/go.mod h1:JaY6n2sDr+z2WTsXkOmNRUfDy6FN0L6Nk7x06ndm4tY= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -1131,8 +1128,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/thanos-io/thanos v0.8.1-0.20200109203923-552ffa4c1a0d/go.mod h1:usT/TxtJQ7DzinTt+G9kinDQmRS5sxwu0unVKZ9vdcw= -github.com/thanos-io/thanos v0.12.3-0.20200603113103-6d3e730e154d h1:Jx/V3MuNqMv/D4k4bHStAQOArnzmB7gnDLXl4boV/Pg= -github.com/thanos-io/thanos v0.12.3-0.20200603113103-6d3e730e154d/go.mod h1:v9d5kS0JbwiGgEmN9BBvAKsn0yBZd9ol1N9X+PS6C6w= +github.com/thanos-io/thanos v0.12.3-0.20200618165043-6c513e5f5c5f h1:UnMVEOejh6tWKUag5tuC0WjKfKmGwJ2+ky0MV4KM52I= +github.com/thanos-io/thanos v0.12.3-0.20200618165043-6c513e5f5c5f/go.mod h1:CPqrM/ibNtlraee0to4dSRiTs+KLI1c3agMS2lmJpz0= github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= @@ -1146,8 +1143,6 @@ github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqri github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-client-go v2.20.1+incompatible h1:HgqpYBng0n7tLJIlyT4kPCIv5XgCsF+kai1NnnrJzEU= github.com/uber/jaeger-client-go v2.20.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= -github.com/uber/jaeger-client-go v2.23.0+incompatible h1:o2g11IUBdEsSZVzF3k7+bahLmxRP/dbOoW4zQ30UlKE= -github.com/uber/jaeger-client-go v2.23.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-client-go v2.23.1+incompatible h1:uArBYHQR0HqLFFAypI7RsWTzPSj/bDpmZZuQjMLSg1A= github.com/uber/jaeger-client-go v2.23.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v1.5.1-0.20181102163054-1fc5c315e03c/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= @@ -1184,8 +1179,8 @@ go.elastic.co/apm/module/apmot v1.5.0/go.mod h1:d2KYwhJParTpyw2WnTNy8geNlHKKFX+4 go.elastic.co/fastjson v1.0.0/go.mod h1:PmeUOMMtLHQr9ZS9J9owrAVg0FkaZDRZJEFTTGHtchs= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= -go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.5-0.20200615073812-232d8fc87f50 h1:ASw9n1EHMftwnP3Az4XW6e308+gNsrHzmdhd0Olz9Hs= +go.etcd.io/bbolt v1.3.5-0.20200615073812-232d8fc87f50/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.0.0-20190709142735-eb7dd97135a5/go.mod h1:N0RPWo9FXJYZQI4BTkDtQylrstIigYHeR18ONnyTufk= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f h1:pBCD+Z7cy5WPTq+R6MmJJvDRpn88cp7bmTypBsn91g4= @@ -1212,7 +1207,6 @@ go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM= go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -1220,7 +1214,6 @@ go.uber.org/automaxprocs v1.2.0/go.mod h1:YfO3fm683kQpzETxlTGZhGIVmXAhaw3gxeBADb go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.4.0 h1:f3WCSC2KzAcBXGATIxAB1E2XuCpNU255wNKZ505qi3E= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -1229,7 +1222,6 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9E go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0 h1:nR6NoDBgAf67s68NhaXbsojM+2gxp3S1hWkHDl27pVU= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= @@ -1285,7 +1277,6 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367 h1:0IiAsCRByjO2QjX7ZPkw5oU9x+n1YqRL802rjC0c3Aw= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= @@ -1339,8 +1330,6 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200421231249-e086a090c8fd h1:QPwSajcTUrFriMF1nJ3XzgoqakqQEsnZf9LdXdi2nkI= -golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1422,15 +1411,14 @@ golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200217220822-9197077df867 h1:JoRuNIf+rpHl+VhScRQQvzbHed86tKkqwPMV34T8myw= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1503,8 +1491,6 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200216192241-b320d3a0f5a2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200422205258-72e4a01eba43 h1:Lcsc5ErIWemp8qAbYffG5vPrqjJ0zk82RTFGifeS1Pc= -golang.org/x/tools v0.0.0-20200422205258-72e4a01eba43/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200603131246-cc40288be839 h1:SxYgZ5FbVts/fm9UsuLycOG8MRWJPm7krdhgPQSayUs= golang.org/x/tools v0.0.0-20200603131246-cc40288be839/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1531,8 +1517,6 @@ google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0 h1:J1Pl9P2lnmYFSJvgs70DKELqHNh8CNWXPbud4njEE2s= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.26.0 h1:VJZ8h6E8ip82FRpQl848c5vAadxlTXrUh8RzQzSRm08= google.golang.org/api v0.26.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -1571,8 +1555,6 @@ google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4 google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200420144010-e5e8543f8aeb h1:nAFaltAMbNVA0rixtwvdnqgSVLX3HFUUvMkEklmzbYM= -google.golang.org/genproto v0.0.0-20200420144010-e5e8543f8aeb/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200603110839-e855014d5736 h1:+IE3xTD+6Eb7QWG5JFp+dQr/XjKpjmrNkh4pdjTdHEs= @@ -1592,20 +1574,16 @@ google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.0 h1:2pJjwYOdkZ9HlN4sWRYBg9ttH5bCOlsueaM+b/oYjwo= -google.golang.org/grpc v1.29.0/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= @@ -1647,14 +1625,11 @@ gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2 h1:XZx7nhd5GMaZpmDaEHFVafUZC7ya0fuo7cSJ3UCKYmM= gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200504163728-5308cda29e3d h1:6bgL440VgqcwXl4vE9sB7DRHhANM+WO/1llfOXSU9ZM= -gopkg.in/yaml.v3 v3.0.0-20200504163728-5308cda29e3d/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200603094226-e3079894b1e8 h1:jL/vaozO53FMfZLySWM+4nulF3gQEC6q5jH90LPomDo= gopkg.in/yaml.v3 v3.0.0-20200603094226-e3079894b1e8/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= @@ -1673,16 +1648,12 @@ k8s.io/api v0.0.0-20190813020757-36bff7324fb7 h1:4uJOjRn9kWq4AqJRE8+qzmAy+lJd9rh k8s.io/api v0.0.0-20190813020757-36bff7324fb7/go.mod h1:3Iy+myeAORNCLgjd/Xu9ebwN7Vh59Bw0vh9jhoX+V58= k8s.io/api v0.0.0-20191115095533-47f6de673b26 h1:6L7CEQVcduEr9eUPN3r3RliLvDrvcaniFOE5B5zRfmc= k8s.io/api v0.0.0-20191115095533-47f6de673b26/go.mod h1:iA/8arsvelvo4IDqIhX4IbjTEKBGgvsf2OraTuRtLFU= -k8s.io/api v0.17.5 h1:EkVieIbn1sC8YCDwckLKLpf+LoVofXYW72+LTZWo4aQ= -k8s.io/api v0.17.5/go.mod h1:0zV5/ungglgy2Rlm3QK8fbxkXVs+BSJWpJP/+8gUVLY= k8s.io/api v0.18.3 h1:2AJaUQdgUZLoDZHrun21PW2Nx9+ll6cUzvn3IKhSIn0= k8s.io/api v0.18.3/go.mod h1:UOaMwERbqJMfeeeHc8XJKawj4P9TgDRnViIqqBeH2QA= k8s.io/apimachinery v0.0.0-20190809020650-423f5d784010 h1:pyoq062NftC1y/OcnbSvgolyZDJ8y4fmUPWMkdA6gfU= k8s.io/apimachinery v0.0.0-20190809020650-423f5d784010/go.mod h1:Waf/xTS2FGRrgXCkO5FP3XxTOWh0qLf2QhL1qFZZ/R8= k8s.io/apimachinery v0.0.0-20191115015347-3c7067801da2 h1:TSH6UZ+y3etc/aDbVqow1NT8o7SJXkxhLKbp3Ywhyvg= k8s.io/apimachinery v0.0.0-20191115015347-3c7067801da2/go.mod h1:dXFS2zaQR8fyzuvRdJDHw2Aerij/yVGJSre0bZQSVJA= -k8s.io/apimachinery v0.17.5 h1:QAjfgeTtSGksdkgyaPrIb4lhU16FWMIzxKejYD5S0gc= -k8s.io/apimachinery v0.17.5/go.mod h1:ioIo1G/a+uONV7Tv+ZmCbMG1/a3kVw5YcDdncd8ugQ0= k8s.io/apimachinery v0.18.3 h1:pOGcbVAhxADgUYnjS08EFXs9QMl8qaH5U4fr5LGUrSk= k8s.io/apimachinery v0.18.3/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= k8s.io/client-go v0.18.3 h1:QaJzz92tsN67oorwzmoB0a9r9ZVHuD5ryjbCKP0U22k= @@ -1701,8 +1672,6 @@ k8s.io/kube-openapi v0.0.0-20190709113604-33be087ad058/go.mod h1:nfDlWeOsu3pUf4y k8s.io/kube-openapi v0.0.0-20190722073852-5e22f3d471e6/go.mod h1:RZvgC8MSN6DjiMV6oIfEE9pDL9CYXokkfaCKZeHm3nc= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/kube-openapi v0.0.0-20200316234421-82d701f24f9d h1:jocF7XFucw2pEiv2wS7wk2FRFCjDFGV1oa4TMs0SAT0= -k8s.io/kube-openapi v0.0.0-20200316234421-82d701f24f9d/go.mod h1:F+5wygcW0wmRTnM3cOgIqGivxkwSWIWT5YdsDbeAOaU= k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6 h1:Oh3Mzx5pJ+yIumsAD0MOECPVeXsVot0UkiaCGVyfGQY= k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/utils v0.0.0-20190809000727-6c36bc71fc4a/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= @@ -1718,7 +1687,6 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e h1:4Z09Hglb792X0kfOBBJUPFEyvVfQWrYT/l8h5EKA6JQ= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= -sigs.k8s.io/structured-merge-diff/v2 v2.0.1/go.mod h1:Wb7vfKAodbKgf6tn1Kl0VvGj7mRH6DGaRcixXEJXTsE= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= diff --git a/loki-build-image/Dockerfile b/loki-build-image/Dockerfile index aeca9cbc21ac4..abba84fe825d4 100644 --- a/loki-build-image/Dockerfile +++ b/loki-build-image/Dockerfile @@ -17,10 +17,10 @@ RUN apk add --no-cache docker-cli # TODO this should be fixed to download and extract the specific release binary from github as we do for golangci and helm above # however we need a commit which hasn't been released yet: https://github.com/drone/drone-cli/commit/1fad337d74ca0ecf420993d9d2d7229a1c99f054 # Read the comment below regarding GO111MODULE=on and why it is necessary -FROM golang:1.13.4 as drone +FROM golang:1.14.2 as drone RUN GO111MODULE=on go get github.com/drone/drone-cli/drone@1fad337d74ca0ecf420993d9d2d7229a1c99f054 -FROM golang:1.13.4-stretch +FROM golang:1.14.2-stretch RUN apt-get update && \ apt-get install -qy \ musl gnupg \ diff --git a/vendor/github.com/cortexproject/cortex/pkg/alertmanager/multitenant.go b/vendor/github.com/cortexproject/cortex/pkg/alertmanager/multitenant.go index 4daad12f73b69..92c129b5fc3fc 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/alertmanager/multitenant.go +++ b/vendor/github.com/cortexproject/cortex/pkg/alertmanager/multitenant.go @@ -481,7 +481,7 @@ func (am *MultitenantAlertmanager) ServeHTTP(w http.ResponseWriter, req *http.Re am.alertmanagersMtx.Unlock() if !ok || !userAM.IsActive() { - http.Error(w, fmt.Sprintf("no Alertmanager for this user ID"), http.StatusNotFound) + http.Error(w, "no Alertmanager for this user ID", http.StatusNotFound) return } userAM.mux.ServeHTTP(w, req) diff --git a/vendor/github.com/cortexproject/cortex/pkg/api/api.go b/vendor/github.com/cortexproject/cortex/pkg/api/api.go index 55348a5ab6afb..b78a8a3a09994 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/api/api.go +++ b/vendor/github.com/cortexproject/cortex/pkg/api/api.go @@ -7,6 +7,7 @@ import ( "net/http" "regexp" "strings" + "time" "github.com/opentracing-contrib/go-stdlib/nethttp" "github.com/opentracing/opentracing-go" @@ -14,6 +15,7 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" + "github.com/gorilla/mux" "github.com/prometheus/common/route" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/promql" @@ -22,8 +24,6 @@ import ( "github.com/weaveworks/common/middleware" "github.com/weaveworks/common/server" - "github.com/gorilla/mux" - "github.com/cortexproject/cortex/pkg/alertmanager" "github.com/cortexproject/cortex/pkg/chunk/purger" "github.com/cortexproject/cortex/pkg/compactor" @@ -181,8 +181,8 @@ func (a *API) RegisterIngester(i *ingester.Ingester, pushConfig distributor.Conf // RegisterPurger registers the endpoints associated with the Purger/DeleteStore. They do not exactly // match the Prometheus API but mirror it closely enough to justify their routing under the Prometheus // component/ -func (a *API) RegisterPurger(store *purger.DeleteStore) { - deleteRequestHandler := purger.NewDeleteRequestHandler(store, prometheus.DefaultRegisterer) +func (a *API) RegisterPurger(store *purger.DeleteStore, deleteRequestCancelPeriod time.Duration) { + deleteRequestHandler := purger.NewDeleteRequestHandler(store, deleteRequestCancelPeriod, prometheus.DefaultRegisterer) a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/admin/tsdb/delete_series", http.HandlerFunc(deleteRequestHandler.AddDeleteRequestHandler), true, "PUT", "POST") a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/admin/tsdb/delete_series", http.HandlerFunc(deleteRequestHandler.GetAllDeleteRequestsHandler), true, "GET") @@ -252,7 +252,14 @@ func (a *API) RegisterCompactor(c *compactor.Compactor) { // RegisterQuerier registers the Prometheus routes supported by the // Cortex querier service. Currently this can not be registered simultaneously // with the QueryFrontend. -func (a *API) RegisterQuerier(queryable storage.Queryable, engine *promql.Engine, distributor *distributor.Distributor, registerRoutesExternally bool, tombstonesLoader *purger.TombstonesLoader) http.Handler { +func (a *API) RegisterQuerier( + queryable storage.Queryable, + engine *promql.Engine, + distributor *distributor.Distributor, + registerRoutesExternally bool, + tombstonesLoader *purger.TombstonesLoader, + querierRequestDuration *prometheus.HistogramVec, +) http.Handler { api := v1.NewAPI( engine, queryable, @@ -288,10 +295,17 @@ func (a *API) RegisterQuerier(queryable storage.Queryable, engine *promql.Engine router = a.server.HTTP } + // Use a separate metric for the querier in order to differentiate requests from the query-frontend when + // running Cortex as a single binary. + inst := middleware.Instrument{ + Duration: querierRequestDuration, + RouteMatcher: router, + } + promRouter := route.New().WithPrefix(a.cfg.ServerPrefix + a.cfg.PrometheusHTTPPrefix + "/api/v1") api.Register(promRouter) cacheGenHeaderMiddleware := getHTTPCacheGenNumberHeaderSetterMiddleware(tombstonesLoader) - promHandler := fakeRemoteAddr(cacheGenHeaderMiddleware.Wrap(promRouter)) + promHandler := fakeRemoteAddr(inst.Wrap(cacheGenHeaderMiddleware.Wrap(promRouter))) a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/read", querier.RemoteReadHandler(queryable), true, "POST") a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/query", promHandler, true, "GET", "POST") @@ -305,7 +319,7 @@ func (a *API) RegisterQuerier(queryable storage.Queryable, engine *promql.Engine legacyPromRouter := route.New().WithPrefix(a.cfg.ServerPrefix + a.cfg.LegacyHTTPPrefix + "/api/v1") api.Register(legacyPromRouter) - legacyPromHandler := fakeRemoteAddr(cacheGenHeaderMiddleware.Wrap(legacyPromRouter)) + legacyPromHandler := fakeRemoteAddr(inst.Wrap(cacheGenHeaderMiddleware.Wrap(legacyPromRouter))) a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/read", querier.RemoteReadHandler(queryable), true, "POST") a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/query", legacyPromHandler, true, "GET", "POST") @@ -331,31 +345,34 @@ func (a *API) RegisterQuerier(queryable storage.Queryable, engine *promql.Engine })) } +// registerQueryAPI registers the Prometheus routes supported by the +// Cortex querier service. Currently this can not be registered simultaneously +// with the Querier. +func (a *API) registerQueryAPI(handler http.Handler) { + a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/read", handler, true, "POST") + a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/query", handler, true, "GET", "POST") + a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/query_range", handler, true, "GET", "POST") + a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/labels", handler, true, "GET", "POST") + a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/label/{name}/values", handler, true, "GET") + a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/series", handler, true, "GET", "POST", "DELETE") + a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/metadata", handler, true, "GET") + + // Register Legacy Routers + a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/read", handler, true, "POST") + a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/query", handler, true, "GET", "POST") + a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/query_range", handler, true, "GET", "POST") + a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/labels", handler, true, "GET", "POST") + a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/label/{name}/values", handler, true, "GET") + a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/series", handler, true, "GET", "POST", "DELETE") + a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/metadata", handler, true, "GET") +} + // RegisterQueryFrontend registers the Prometheus routes supported by the // Cortex querier service. Currently this can not be registered simultaneously // with the Querier. func (a *API) RegisterQueryFrontend(f *frontend.Frontend) { frontend.RegisterFrontendServer(a.server.GRPC, f) - - // Previously the frontend handled all calls to the provided prefix. Instead explicit - // routing is used since it will be required to enable the frontend to be run as part - // of a single binary in the future. - a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/read", f.Handler(), true, "POST") - a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/query", f.Handler(), true, "GET", "POST") - a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/query_range", f.Handler(), true, "GET", "POST") - a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/labels", f.Handler(), true, "GET", "POST") - a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/label/{name}/values", f.Handler(), true, "GET") - a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/series", f.Handler(), true, "GET", "POST", "DELETE") - a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/metadata", f.Handler(), true, "GET") - - // Register Legacy Routers - a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/read", f.Handler(), true, "POST") - a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/query", f.Handler(), true, "GET", "POST") - a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/query_range", f.Handler(), true, "GET", "POST") - a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/labels", f.Handler(), true, "GET", "POST") - a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/label/{name}/values", f.Handler(), true, "GET") - a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/series", f.Handler(), true, "GET", "POST", "DELETE") - a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/metadata", f.Handler(), true, "GET") + a.registerQueryAPI(f.Handler()) } // RegisterServiceMapHandler registers the Cortex structs service handler diff --git a/vendor/github.com/cortexproject/cortex/pkg/chunk/aws/dynamodb_storage_client.go b/vendor/github.com/cortexproject/cortex/pkg/chunk/aws/dynamodb_storage_client.go index c5131ddfd34c7..6d7ca71fb46e8 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/chunk/aws/dynamodb_storage_client.go +++ b/vendor/github.com/cortexproject/cortex/pkg/chunk/aws/dynamodb_storage_client.go @@ -576,8 +576,8 @@ func processChunkResponse(response *dynamodb.BatchGetItemOutput, chunksByKey map // PutChunkAndIndex implements chunk.ObjectAndIndexClient // Combine both sets of writes before sending to DynamoDB, for performance -func (a dynamoDBStorageClient) PutChunkAndIndex(ctx context.Context, c chunk.Chunk, index chunk.WriteBatch) error { - dynamoDBWrites, err := a.writesForChunks([]chunk.Chunk{c}) +func (a dynamoDBStorageClient) PutChunksAndIndex(ctx context.Context, chunks []chunk.Chunk, index chunk.WriteBatch) error { + dynamoDBWrites, err := a.writesForChunks(chunks) if err != nil { return err } diff --git a/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go b/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go index 9ada6b61602f6..994c49b745239 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go +++ b/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go @@ -96,12 +96,12 @@ func (c *backgroundCache) Store(ctx context.Context, keys []string, bufs [][]byt } select { case c.bgWrites <- bgWrite: - c.queueLength.Add(float64(len(keys))) + c.queueLength.Add(float64(num)) default: - c.droppedWriteBack.Add(float64(len(keys))) + c.droppedWriteBack.Add(float64(num)) sp := opentracing.SpanFromContext(ctx) if sp != nil { - sp.LogFields(otlog.Int("dropped", len(keys))) + sp.LogFields(otlog.Int("dropped", num)) } return // queue is full; give up } diff --git a/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached_client.go b/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached_client.go index c468b35655f54..df0969dc78f3b 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached_client.go +++ b/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached_client.go @@ -12,6 +12,7 @@ import ( "github.com/bradfitz/gomemcache/memcache" "github.com/go-kit/kit/log/level" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/thanos-io/thanos/pkg/discovery/dns" @@ -128,6 +129,22 @@ func (c *memcachedClient) Stop() { c.wait.Wait() } +func (c *memcachedClient) Set(item *memcache.Item) error { + err := c.Client.Set(item) + if err == nil { + return nil + } + + // Inject the server address in order to have more information about which memcached + // backend server failed. This is a best effort. + addr, addrErr := c.serverList.PickServer(item.Key) + if addrErr != nil { + return err + } + + return errors.Wrapf(err, "server=%s", addr) +} + func (c *memcachedClient) updateLoop(updateInterval time.Duration) { defer c.wait.Done() ticker := time.NewTicker(updateInterval) diff --git a/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store.go b/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store.go index bd69eedeea384..41eef9c41be51 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store.go +++ b/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store.go @@ -63,9 +63,8 @@ type StoreConfig struct { // is set, use different caches for ingesters and queriers. chunkCacheStubs bool // don't write the full chunk to cache, just a stub entry - // When DisableChunksDeduplication is true, cache would not be checked for whether chunk is already written. - // It would still write the chunk back to cache for reads. - DisableChunksDeduplication bool `yaml:"-"` + // When DisableIndexDeduplication is true and chunk is already there in cache, only index would be written to the store and not chunk. + DisableIndexDeduplication bool `yaml:"-"` } // RegisterFlags adds the flags required to config this to the given FlagSet @@ -447,7 +446,7 @@ func (c *store) lookupChunksByMetricName(ctx context.Context, userID string, fro } func (c *baseStore) lookupIdsByMetricNameMatcher(ctx context.Context, from, through model.Time, userID, metricName string, matcher *labels.Matcher, filter func([]IndexQuery) []IndexQuery) ([]string, error) { - log, ctx := spanlogger.New(ctx, "Store.lookupIdsByMetricNameMatcher", "metricName", metricName, "matcher", matcher) + log, ctx := spanlogger.New(ctx, "Store.lookupIdsByMetricNameMatcher", "metricName", metricName, "matcher", formatMatcher(matcher)) defer log.Span.Finish() var err error @@ -475,11 +474,11 @@ func (c *baseStore) lookupIdsByMetricNameMatcher(ctx context.Context, from, thro if err != nil { return nil, err } - level.Debug(log).Log("matcher", matcher, "queries", len(queries)) + level.Debug(log).Log("matcher", formatMatcher(matcher), "queries", len(queries)) if filter != nil { queries = filter(queries) - level.Debug(log).Log("matcher", matcher, "filteredQueries", len(queries)) + level.Debug(log).Log("matcher", formatMatcher(matcher), "filteredQueries", len(queries)) } entries, err := c.lookupEntriesByQueries(ctx, queries) @@ -490,17 +489,27 @@ func (c *baseStore) lookupIdsByMetricNameMatcher(ctx context.Context, from, thro } else if err != nil { return nil, err } - level.Debug(log).Log("matcher", matcher, "entries", len(entries)) + level.Debug(log).Log("matcher", formatMatcher(matcher), "entries", len(entries)) ids, err := c.parseIndexEntries(ctx, entries, matcher) if err != nil { return nil, err } - level.Debug(log).Log("matcher", matcher, "ids", len(ids)) + level.Debug(log).Log("matcher", formatMatcher(matcher), "ids", len(ids)) return ids, nil } +// Using this function avoids logging of nil matcher, which works, but indirectly via panic and recover. +// That confuses attached debugger, which wants to breakpoint on each panic. +// Using simple check is also faster. +func formatMatcher(matcher *labels.Matcher) string { + if matcher == nil { + return "nil" + } + return matcher.String() +} + func (c *baseStore) lookupEntriesByQueries(ctx context.Context, queries []IndexQuery) ([]IndexEntry, error) { log, ctx := spanlogger.New(ctx, "store.lookupEntriesByQueries") defer log.Span.Finish() diff --git a/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/factory.go b/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/factory.go index 95f2a61ccdc18..e69a51a2229f8 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/factory.go +++ b/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/factory.go @@ -15,15 +15,13 @@ type Config struct{} var ( // DefaultEncoding exported for use in unit tests elsewhere - DefaultEncoding = Bigchunk - alwaysMarshalFullsizeChunks = true - bigchunkSizeCapBytes = 0 + DefaultEncoding = Bigchunk + bigchunkSizeCapBytes = 0 ) // RegisterFlags registers configuration settings. func (Config) RegisterFlags(f *flag.FlagSet) { f.Var(&DefaultEncoding, "ingester.chunk-encoding", "Encoding version to use for chunks.") - flag.BoolVar(&alwaysMarshalFullsizeChunks, "store.fullsize-chunks", alwaysMarshalFullsizeChunks, "When saving varbit chunks, pad to 1024 bytes") flag.IntVar(&bigchunkSizeCapBytes, "store.bigchunk-size-cap-bytes", bigchunkSizeCapBytes, "When using bigchunk encoding, start a new bigchunk if over this size (0 = unlimited)") } diff --git a/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/varbit.go b/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/varbit.go index c9580214d2cdd..a9d1c2f28771d 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/varbit.go +++ b/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/varbit.go @@ -318,6 +318,7 @@ func (c varbitChunk) Utilization() float64 { } // marshalLen returns the number of bytes that should be marshalled for this chunk +// (if someone has used a version of this code that doesn't just send 1024 every time) func (c varbitChunk) marshalLen() int { bits := c.nextSampleOffset() if bits < varbitThirdSampleBitOffset { @@ -340,10 +341,7 @@ func (c varbitChunk) Len() int { } func (c varbitChunk) Size() int { - if alwaysMarshalFullsizeChunks { - return cap(c) - } - return c.marshalLen() + return cap(c) } func (c varbitChunk) firstTime() model.Time { diff --git a/vendor/github.com/cortexproject/cortex/pkg/chunk/inmemory_storage_client.go b/vendor/github.com/cortexproject/cortex/pkg/chunk/inmemory_storage_client.go index 5a3aed3b6db38..4c3a5bd6a427a 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/chunk/inmemory_storage_client.go +++ b/vendor/github.com/cortexproject/cortex/pkg/chunk/inmemory_storage_client.go @@ -20,7 +20,8 @@ type MockStorage struct { tables map[string]*mockTable objects map[string][]byte - numWrites int + numIndexWrites int + numChunkWrites int } type mockTable struct { @@ -137,7 +138,7 @@ func (m *MockStorage) BatchWrite(ctx context.Context, batch WriteBatch) error { mockBatch := *batch.(*mockWriteBatch) seenWrites := map[string]bool{} - m.numWrites += len(mockBatch.inserts) + m.numIndexWrites += len(mockBatch.inserts) for _, req := range mockBatch.inserts { table, ok := m.tables[req.tableName] @@ -301,6 +302,8 @@ func (m *MockStorage) PutChunks(_ context.Context, chunks []Chunk) error { m.mtx.Lock() defer m.mtx.Unlock() + m.numChunkWrites += len(chunks) + for i := range chunks { buf, err := chunks[i].Encoded() if err != nil { diff --git a/vendor/github.com/cortexproject/cortex/pkg/chunk/local/boltdb_index_client.go b/vendor/github.com/cortexproject/cortex/pkg/chunk/local/boltdb_index_client.go index 8dad54ee5e100..0ef1a401a1451 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/chunk/local/boltdb_index_client.go +++ b/vendor/github.com/cortexproject/cortex/pkg/chunk/local/boltdb_index_client.go @@ -8,6 +8,7 @@ import ( "fmt" "os" "path" + "path/filepath" "sync" "time" @@ -127,9 +128,8 @@ func (b *BoltIndexClient) Stop() { } func (b *BoltIndexClient) NewWriteBatch() chunk.WriteBatch { - return &boltWriteBatch{ - puts: map[string]map[string][]byte{}, - deletes: map[string]map[string]struct{}{}, + return &BoltWriteBatch{ + Writes: map[string]TableWrites{}, } } @@ -171,52 +171,49 @@ func (b *BoltIndexClient) GetDB(name string, operation int) (*bbolt.DB, error) { return db, nil } -func (b *BoltIndexClient) BatchWrite(ctx context.Context, batch chunk.WriteBatch) error { - // ToDo: too much code duplication, refactor this - for table, kvps := range batch.(*boltWriteBatch).puts { - db, err := b.GetDB(table, DBOperationWrite) - if err != nil { - return err - } +func (b *BoltIndexClient) WriteToDB(ctx context.Context, db *bbolt.DB, writes TableWrites) error { + return db.Update(func(tx *bbolt.Tx) error { + var b *bbolt.Bucket - if err := db.Update(func(tx *bbolt.Tx) error { - b, err := tx.CreateBucketIfNotExists(bucketName) + // a bucket should already exist for deletes, for other writes we create one otherwise. + if len(writes.deletes) != 0 { + b = tx.Bucket(bucketName) + if b == nil { + return fmt.Errorf("bucket %s not found in table %s", bucketName, filepath.Base(db.Path())) + } + } else { + var err error + b, err = tx.CreateBucketIfNotExists(bucketName) if err != nil { return err } + } - for key, value := range kvps { - if err := b.Put([]byte(key), value); err != nil { - return err - } + for key, value := range writes.puts { + if err := b.Put([]byte(key), value); err != nil { + return err } + } - return nil - }); err != nil { - return err + for key := range writes.deletes { + if err := b.Delete([]byte(key)); err != nil { + return err + } } - } - for table, kvps := range batch.(*boltWriteBatch).deletes { + return nil + }) +} + +func (b *BoltIndexClient) BatchWrite(ctx context.Context, batch chunk.WriteBatch) error { + for table, writes := range batch.(*BoltWriteBatch).Writes { db, err := b.GetDB(table, DBOperationWrite) if err != nil { return err } - if err := db.Update(func(tx *bbolt.Tx) error { - b := tx.Bucket(bucketName) - if b == nil { - return fmt.Errorf("Bucket %s not found in table %s", bucketName, table) - } - - for key := range kvps { - if err := b.Delete([]byte(key)); err != nil { - return err - } - } - - return nil - }); err != nil { + err = b.WriteToDB(ctx, db, writes) + if err != nil { return err } } @@ -285,31 +282,40 @@ func (b *BoltIndexClient) QueryDB(ctx context.Context, db *bbolt.DB, query chunk }) } -type boltWriteBatch struct { - puts map[string]map[string][]byte - deletes map[string]map[string]struct{} +type TableWrites struct { + puts map[string][]byte + deletes map[string]struct{} } -func (b *boltWriteBatch) Delete(tableName, hashValue string, rangeValue []byte) { - table, ok := b.deletes[tableName] +type BoltWriteBatch struct { + Writes map[string]TableWrites +} + +func (b *BoltWriteBatch) getOrCreateTableWrites(tableName string) TableWrites { + writes, ok := b.Writes[tableName] if !ok { - table = map[string]struct{}{} - b.deletes[tableName] = table + writes = TableWrites{ + puts: map[string][]byte{}, + deletes: map[string]struct{}{}, + } + b.Writes[tableName] = writes } + return writes +} + +func (b *BoltWriteBatch) Delete(tableName, hashValue string, rangeValue []byte) { + writes := b.getOrCreateTableWrites(tableName) + key := hashValue + separator + string(rangeValue) - table[key] = struct{}{} + writes.deletes[key] = struct{}{} } -func (b *boltWriteBatch) Add(tableName, hashValue string, rangeValue []byte, value []byte) { - table, ok := b.puts[tableName] - if !ok { - table = map[string][]byte{} - b.puts[tableName] = table - } +func (b *BoltWriteBatch) Add(tableName, hashValue string, rangeValue []byte, value []byte) { + writes := b.getOrCreateTableWrites(tableName) key := hashValue + separator + string(rangeValue) - table[key] = value + writes.puts[key] = value } type boltReadBatch struct { diff --git a/vendor/github.com/cortexproject/cortex/pkg/chunk/purger/delete_requests_store.go b/vendor/github.com/cortexproject/cortex/pkg/chunk/purger/delete_requests_store.go index 1d156caa78c2c..4f8d1025f37d9 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/chunk/purger/delete_requests_store.go +++ b/vendor/github.com/cortexproject/cortex/pkg/chunk/purger/delete_requests_store.go @@ -97,6 +97,12 @@ func NewDeleteStore(cfg DeleteStoreConfig, indexClient chunk.IndexClient) (*Dele // Add creates entries for a new delete request. func (ds *DeleteStore) AddDeleteRequest(ctx context.Context, userID string, startTime, endTime model.Time, selectors []string) error { + return ds.addDeleteRequest(ctx, userID, model.Now(), startTime, endTime, selectors) + +} + +// addDeleteRequest is also used for tests to create delete requests with different createdAt time. +func (ds *DeleteStore) addDeleteRequest(ctx context.Context, userID string, createdAt, startTime, endTime model.Time, selectors []string) error { requestID := generateUniqueID(userID, selectors) for { @@ -122,7 +128,7 @@ func (ds *DeleteStore) AddDeleteRequest(ctx context.Context, userID string, star writeBatch.Add(ds.cfg.RequestsTableName, string(deleteRequestID), []byte(userIDAndRequestID), []byte(StatusReceived)) // Add another entry with additional details like creation time, time range of delete request and selectors in value - rangeValue := fmt.Sprintf("%x:%x:%x", int64(model.Now()), int64(startTime), int64(endTime)) + rangeValue := fmt.Sprintf("%x:%x:%x", int64(createdAt), int64(startTime), int64(endTime)) writeBatch.Add(ds.cfg.RequestsTableName, fmt.Sprintf("%s:%s", deleteRequestDetails, userIDAndRequestID), []byte(rangeValue), []byte(strings.Join(selectors, separator))) diff --git a/vendor/github.com/cortexproject/cortex/pkg/chunk/purger/purger.go b/vendor/github.com/cortexproject/cortex/pkg/chunk/purger/purger.go index 8b463ef89ad2e..7a771e0aa34e3 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/chunk/purger/purger.go +++ b/vendor/github.com/cortexproject/cortex/pkg/chunk/purger/purger.go @@ -26,14 +26,18 @@ import ( ) const ( - millisecondPerDay = int64(24 * time.Hour / time.Millisecond) - deleteRequestCancellationDeadline = 24 * time.Hour + millisecondPerDay = int64(24 * time.Hour / time.Millisecond) + statusSuccess = "success" + statusFail = "fail" ) type purgerMetrics struct { - deleteRequestsProcessedTotal *prometheus.CounterVec - deleteRequestsChunksSelectedTotal *prometheus.CounterVec - deleteRequestsProcessingFailures *prometheus.CounterVec + deleteRequestsProcessedTotal *prometheus.CounterVec + deleteRequestsChunksSelectedTotal *prometheus.CounterVec + deleteRequestsProcessingFailures *prometheus.CounterVec + loadPendingRequestsAttempsTotal *prometheus.CounterVec + oldestPendingDeleteRequestAgeSeconds prometheus.Gauge + pendingDeleteRequestsCount prometheus.Gauge } func newPurgerMetrics(r prometheus.Registerer) *purgerMetrics { @@ -54,6 +58,21 @@ func newPurgerMetrics(r prometheus.Registerer) *purgerMetrics { Name: "purger_delete_requests_processing_failures_total", Help: "Number of delete requests processing failures per user", }, []string{"user"}) + m.loadPendingRequestsAttempsTotal = promauto.With(r).NewCounterVec(prometheus.CounterOpts{ + Namespace: "cortex", + Name: "purger_load_pending_requests_attempts_total", + Help: "Number of attempts that were made to load pending requests with status", + }, []string{"status"}) + m.oldestPendingDeleteRequestAgeSeconds = promauto.With(r).NewGauge(prometheus.GaugeOpts{ + Namespace: "cortex", + Name: "purger_oldest_pending_delete_request_age_seconds", + Help: "Age of oldest pending delete request in seconds", + }) + m.pendingDeleteRequestsCount = promauto.With(r).NewGauge(prometheus.GaugeOpts{ + Namespace: "cortex", + Name: "purger_pending_delete_requests_count", + Help: "Count of requests which are in process or are ready to be processed", + }) return &m } @@ -63,11 +82,12 @@ type deleteRequestWithLogger struct { logger log.Logger // logger is initialized with userID and requestID to add context to every log generated using this } -// Config holds config for DataPurger +// Config holds config for Purger type Config struct { - Enable bool `yaml:"enable"` - NumWorkers int `yaml:"num_workers"` - ObjectStoreType string `yaml:"object_store_type"` + Enable bool `yaml:"enable"` + NumWorkers int `yaml:"num_workers"` + ObjectStoreType string `yaml:"object_store_type"` + DeleteRequestCancelPeriod time.Duration `yaml:"delete_request_cancel_period"` } // RegisterFlags registers CLI flags for Config @@ -75,6 +95,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.BoolVar(&cfg.Enable, "purger.enable", false, "Enable purger to allow deletion of series. Be aware that Delete series feature is still experimental") f.IntVar(&cfg.NumWorkers, "purger.num-workers", 2, "Number of workers executing delete plans in parallel") f.StringVar(&cfg.ObjectStoreType, "purger.object-store-type", "", "Name of the object store to use for storing delete plans") + f.DurationVar(&cfg.DeleteRequestCancelPeriod, "purger.delete-request-cancel-period", 24*time.Hour, "Allow cancellation of delete request until duration after they are created. Data would be deleted only after delete requests have been older than this duration. Ideally this should be set to at least 24h.") } type workerJob struct { @@ -84,8 +105,8 @@ type workerJob struct { logger log.Logger } -// DataPurger does the purging of data which is requested to be deleted -type DataPurger struct { +// Purger does the purging of data which is requested to be deleted +type Purger struct { services.Service cfg Config @@ -99,7 +120,7 @@ type DataPurger struct { // we would only allow processing of singe delete request at a time since delete requests touching same chunks could change the chunk IDs of partially deleted chunks // and break the purge plan for other requests - inProcessRequestIDs map[string]string + inProcessRequests map[string]DeleteRequest inProcessRequestIDsMtx sync.RWMutex // We do not want to limit pulling new delete requests to a fixed interval which otherwise would limit number of delete requests we process per user. @@ -115,11 +136,11 @@ type DataPurger struct { wg sync.WaitGroup } -// NewDataPurger creates a new DataPurger -func NewDataPurger(cfg Config, deleteStore *DeleteStore, chunkStore chunk.Store, storageClient chunk.ObjectClient, registerer prometheus.Registerer) (*DataPurger, error) { +// NewPurger creates a new Purger +func NewPurger(cfg Config, deleteStore *DeleteStore, chunkStore chunk.Store, storageClient chunk.ObjectClient, registerer prometheus.Registerer) (*Purger, error) { util.WarnExperimentalUse("Delete series API") - dataPurger := DataPurger{ + purger := Purger{ cfg: cfg, deleteStore: deleteStore, chunkStore: chunkStore, @@ -128,44 +149,52 @@ func NewDataPurger(cfg Config, deleteStore *DeleteStore, chunkStore chunk.Store, pullNewRequestsChan: make(chan struct{}, 1), executePlansChan: make(chan deleteRequestWithLogger, 50), workerJobChan: make(chan workerJob, 50), - inProcessRequestIDs: map[string]string{}, + inProcessRequests: map[string]DeleteRequest{}, usersWithPendingRequests: map[string]struct{}{}, pendingPlansCount: map[string]int{}, } - dataPurger.Service = services.NewBasicService(dataPurger.init, dataPurger.loop, dataPurger.stop) - return &dataPurger, nil + purger.Service = services.NewBasicService(purger.init, purger.loop, purger.stop) + return &purger, nil } // init starts workers, scheduler and then loads in process delete requests -func (dp *DataPurger) init(ctx context.Context) error { - for i := 0; i < dp.cfg.NumWorkers; i++ { - dp.wg.Add(1) - go dp.worker() +func (p *Purger) init(ctx context.Context) error { + for i := 0; i < p.cfg.NumWorkers; i++ { + p.wg.Add(1) + go p.worker() } - dp.wg.Add(1) - go dp.jobScheduler(ctx) + p.wg.Add(1) + go p.jobScheduler(ctx) - return dp.loadInprocessDeleteRequests() + return p.loadInprocessDeleteRequests() } -func (dp *DataPurger) loop(ctx context.Context) error { - loadRequestsTicker := time.NewTicker(time.Hour) - defer loadRequestsTicker.Stop() - +func (p *Purger) loop(ctx context.Context) error { loadRequests := func() { - err := dp.pullDeleteRequestsToPlanDeletes() + status := statusSuccess + + err := p.pullDeleteRequestsToPlanDeletes() if err != nil { + status = statusFail level.Error(util.Logger).Log("msg", "error pulling delete requests for building plans", "err", err) } + + p.metrics.loadPendingRequestsAttempsTotal.WithLabelValues(status).Inc() } + // load requests on startup instead of waiting for first ticker + loadRequests() + + loadRequestsTicker := time.NewTicker(time.Hour) + defer loadRequestsTicker.Stop() + for { select { case <-loadRequestsTicker.C: loadRequests() - case <-dp.pullNewRequestsChan: + case <-p.pullNewRequestsChan: loadRequests() case <-ctx.Done(): return nil @@ -174,102 +203,102 @@ func (dp *DataPurger) loop(ctx context.Context) error { } // Stop waits until all background tasks stop. -func (dp *DataPurger) stop(_ error) error { - dp.wg.Wait() +func (p *Purger) stop(_ error) error { + p.wg.Wait() return nil } -func (dp *DataPurger) workerJobCleanup(job workerJob) { - err := dp.removeDeletePlan(context.Background(), job.userID, job.deleteRequestID, job.planNo) +func (p *Purger) workerJobCleanup(job workerJob) { + err := p.removeDeletePlan(context.Background(), job.userID, job.deleteRequestID, job.planNo) if err != nil { level.Error(job.logger).Log("msg", "error removing delete plan", "plan_no", job.planNo, "err", err) return } - dp.pendingPlansCountMtx.Lock() - dp.pendingPlansCount[job.deleteRequestID]-- + p.pendingPlansCountMtx.Lock() + p.pendingPlansCount[job.deleteRequestID]-- - if dp.pendingPlansCount[job.deleteRequestID] == 0 { + if p.pendingPlansCount[job.deleteRequestID] == 0 { level.Info(job.logger).Log("msg", "finished execution of all plans, cleaning up and updating status of request") - err := dp.deleteStore.UpdateStatus(context.Background(), job.userID, job.deleteRequestID, StatusProcessed) + err := p.deleteStore.UpdateStatus(context.Background(), job.userID, job.deleteRequestID, StatusProcessed) if err != nil { level.Error(job.logger).Log("msg", "error updating delete request status to process", "err", err) } - dp.metrics.deleteRequestsProcessedTotal.WithLabelValues(job.userID).Inc() - delete(dp.pendingPlansCount, job.deleteRequestID) - dp.pendingPlansCountMtx.Unlock() + p.metrics.deleteRequestsProcessedTotal.WithLabelValues(job.userID).Inc() + delete(p.pendingPlansCount, job.deleteRequestID) + p.pendingPlansCountMtx.Unlock() - dp.inProcessRequestIDsMtx.Lock() - delete(dp.inProcessRequestIDs, job.userID) - dp.inProcessRequestIDsMtx.Unlock() + p.inProcessRequestIDsMtx.Lock() + delete(p.inProcessRequests, job.userID) + p.inProcessRequestIDsMtx.Unlock() // request loading of more delete request if // - user has more pending requests and // - we do not have a pending request to load more requests - dp.usersWithPendingRequestsMtx.Lock() - defer dp.usersWithPendingRequestsMtx.Unlock() - if _, ok := dp.usersWithPendingRequests[job.userID]; ok { - delete(dp.usersWithPendingRequests, job.userID) + p.usersWithPendingRequestsMtx.Lock() + defer p.usersWithPendingRequestsMtx.Unlock() + if _, ok := p.usersWithPendingRequests[job.userID]; ok { + delete(p.usersWithPendingRequests, job.userID) select { - case dp.pullNewRequestsChan <- struct{}{}: + case p.pullNewRequestsChan <- struct{}{}: // sent default: // already sent } } } else { - dp.pendingPlansCountMtx.Unlock() + p.pendingPlansCountMtx.Unlock() } } // we send all the delete plans to workerJobChan -func (dp *DataPurger) jobScheduler(ctx context.Context) { - defer dp.wg.Done() +func (p *Purger) jobScheduler(ctx context.Context) { + defer p.wg.Done() for { select { - case req := <-dp.executePlansChan: + case req := <-p.executePlansChan: numPlans := numPlans(req.StartTime, req.EndTime) level.Info(req.logger).Log("msg", "sending jobs to workers for purging data", "num_jobs", numPlans) - dp.pendingPlansCountMtx.Lock() - dp.pendingPlansCount[req.RequestID] = numPlans - dp.pendingPlansCountMtx.Unlock() + p.pendingPlansCountMtx.Lock() + p.pendingPlansCount[req.RequestID] = numPlans + p.pendingPlansCountMtx.Unlock() for i := 0; i < numPlans; i++ { - dp.workerJobChan <- workerJob{planNo: i, userID: req.UserID, + p.workerJobChan <- workerJob{planNo: i, userID: req.UserID, deleteRequestID: req.RequestID, logger: req.logger} } case <-ctx.Done(): - close(dp.workerJobChan) + close(p.workerJobChan) return } } } -func (dp *DataPurger) worker() { - defer dp.wg.Done() +func (p *Purger) worker() { + defer p.wg.Done() - for job := range dp.workerJobChan { - err := dp.executePlan(job.userID, job.deleteRequestID, job.planNo, job.logger) + for job := range p.workerJobChan { + err := p.executePlan(job.userID, job.deleteRequestID, job.planNo, job.logger) if err != nil { - dp.metrics.deleteRequestsProcessingFailures.WithLabelValues(job.userID).Inc() + p.metrics.deleteRequestsProcessingFailures.WithLabelValues(job.userID).Inc() level.Error(job.logger).Log("msg", "error executing delete plan", "plan_no", job.planNo, "err", err) continue } - dp.workerJobCleanup(job) + p.workerJobCleanup(job) } } -func (dp *DataPurger) executePlan(userID, requestID string, planNo int, logger log.Logger) error { +func (p *Purger) executePlan(userID, requestID string, planNo int, logger log.Logger) error { logger = log.With(logger, "plan_no", planNo) - plan, err := dp.getDeletePlan(context.Background(), userID, requestID, planNo) + plan, err := p.getDeletePlan(context.Background(), userID, requestID, planNo) if err != nil { if err == chunk.ErrStorageObjectNotFound { level.Info(logger).Log("msg", "plan not found, must have been executed already") @@ -300,7 +329,7 @@ func (dp *DataPurger) executePlan(userID, requestID string, planNo int, logger l } } - err = dp.chunkStore.DeleteChunk(ctx, chunkRef.From, chunkRef.Through, chunkRef.UserID, + err = p.chunkStore.DeleteChunk(ctx, chunkRef.From, chunkRef.Through, chunkRef.UserID, chunkDetails.ID, client.FromLabelAdaptersToLabels(plan.ChunksGroup[i].Labels), partiallyDeletedInterval) if err != nil { if isMissingChunkErr(err) { @@ -315,7 +344,7 @@ func (dp *DataPurger) executePlan(userID, requestID string, planNo int, logger l level.Debug(logger).Log("msg", "deleting series", "labels", plan.ChunksGroup[i].Labels) // this is mostly required to clean up series ids from series store - err := dp.chunkStore.DeleteSeriesIDs(ctx, model.Time(plan.PlanInterval.StartTimestampMs), model.Time(plan.PlanInterval.EndTimestampMs), + err := p.chunkStore.DeleteSeriesIDs(ctx, model.Time(plan.PlanInterval.StartTimestampMs), model.Time(plan.PlanInterval.EndTimestampMs), userID, client.FromLabelAdaptersToLabels(plan.ChunksGroup[i].Labels)) if err != nil { return err @@ -328,8 +357,8 @@ func (dp *DataPurger) executePlan(userID, requestID string, planNo int, logger l } // we need to load all in process delete requests on startup to finish them first -func (dp *DataPurger) loadInprocessDeleteRequests() error { - requestsWithBuildingPlanStatus, err := dp.deleteStore.GetDeleteRequestsByStatus(context.Background(), StatusBuildingPlan) +func (p *Purger) loadInprocessDeleteRequests() error { + requestsWithBuildingPlanStatus, err := p.deleteStore.GetDeleteRequestsByStatus(context.Background(), StatusBuildingPlan) if err != nil { return err } @@ -339,19 +368,19 @@ func (dp *DataPurger) loadInprocessDeleteRequests() error { level.Info(req.logger).Log("msg", "loaded in process delete requests with status building plan") - dp.inProcessRequestIDs[deleteRequest.UserID] = deleteRequest.RequestID - err := dp.buildDeletePlan(req) + p.inProcessRequests[deleteRequest.UserID] = deleteRequest + err := p.buildDeletePlan(req) if err != nil { - dp.metrics.deleteRequestsProcessingFailures.WithLabelValues(deleteRequest.UserID).Inc() + p.metrics.deleteRequestsProcessingFailures.WithLabelValues(deleteRequest.UserID).Inc() level.Error(req.logger).Log("msg", "error building delete plan", "err", err) continue } level.Info(req.logger).Log("msg", "sending delete request for execution") - dp.executePlansChan <- req + p.executePlansChan <- req } - requestsWithDeletingStatus, err := dp.deleteStore.GetDeleteRequestsByStatus(context.Background(), StatusDeleting) + requestsWithDeletingStatus, err := p.deleteStore.GetDeleteRequestsByStatus(context.Background(), StatusDeleting) if err != nil { return err } @@ -360,8 +389,8 @@ func (dp *DataPurger) loadInprocessDeleteRequests() error { req := makeDeleteRequestWithLogger(deleteRequest, util.Logger) level.Info(req.logger).Log("msg", "loaded in process delete requests with status deleting") - dp.inProcessRequestIDs[deleteRequest.UserID] = deleteRequest.RequestID - dp.executePlansChan <- req + p.inProcessRequests[deleteRequest.UserID] = deleteRequest + p.executePlansChan <- req } return nil @@ -369,51 +398,71 @@ func (dp *DataPurger) loadInprocessDeleteRequests() error { // pullDeleteRequestsToPlanDeletes pulls delete requests which do not have their delete plans built yet and sends them for building delete plans // after pulling delete requests for building plans, it updates its status to StatusBuildingPlan status to avoid picking this up again next time -func (dp *DataPurger) pullDeleteRequestsToPlanDeletes() error { - deleteRequests, err := dp.deleteStore.GetDeleteRequestsByStatus(context.Background(), StatusReceived) +func (p *Purger) pullDeleteRequestsToPlanDeletes() error { + deleteRequests, err := p.deleteStore.GetDeleteRequestsByStatus(context.Background(), StatusReceived) if err != nil { return err } + p.inProcessRequestIDsMtx.RLock() + pendingDeleteRequestsCount := len(p.inProcessRequests) + p.inProcessRequestIDsMtx.RUnlock() + + now := model.Now() + oldestPendingRequestCreatedAt := now + + // requests which are still being processed are also considered pending + if pendingDeleteRequestsCount != 0 { + oldestInProcessRequest := p.getOldestInProcessRequest() + if oldestInProcessRequest != nil { + oldestPendingRequestCreatedAt = oldestInProcessRequest.CreatedAt + } + } + for _, deleteRequest := range deleteRequests { // adding an extra minute here to avoid a race between cancellation of request and picking of the request for processing - if deleteRequest.CreatedAt.Add(deleteRequestCancellationDeadline).Add(time.Minute).After(model.Now()) { + if deleteRequest.CreatedAt.Add(p.cfg.DeleteRequestCancelPeriod).Add(time.Minute).After(model.Now()) { continue } - dp.inProcessRequestIDsMtx.RLock() - inprocessDeleteRequestID := dp.inProcessRequestIDs[deleteRequest.UserID] - dp.inProcessRequestIDsMtx.RUnlock() + pendingDeleteRequestsCount++ + if deleteRequest.CreatedAt.Before(oldestPendingRequestCreatedAt) { + oldestPendingRequestCreatedAt = deleteRequest.CreatedAt + } + + p.inProcessRequestIDsMtx.RLock() + inprocessDeleteRequest, ok := p.inProcessRequests[deleteRequest.UserID] + p.inProcessRequestIDsMtx.RUnlock() - if inprocessDeleteRequestID != "" { - dp.usersWithPendingRequestsMtx.Lock() - dp.usersWithPendingRequests[deleteRequest.UserID] = struct{}{} - dp.usersWithPendingRequestsMtx.Unlock() + if ok { + p.usersWithPendingRequestsMtx.Lock() + p.usersWithPendingRequests[deleteRequest.UserID] = struct{}{} + p.usersWithPendingRequestsMtx.Unlock() level.Debug(util.Logger).Log("msg", "skipping delete request processing for now since another request from same user is already in process", - "inprocess_request_id", inprocessDeleteRequestID, + "inprocess_request_id", inprocessDeleteRequest.RequestID, "skipped_request_id", deleteRequest.RequestID, "user_id", deleteRequest.UserID) continue } - err = dp.deleteStore.UpdateStatus(context.Background(), deleteRequest.UserID, deleteRequest.RequestID, StatusBuildingPlan) + err = p.deleteStore.UpdateStatus(context.Background(), deleteRequest.UserID, deleteRequest.RequestID, StatusBuildingPlan) if err != nil { return err } - dp.inProcessRequestIDsMtx.Lock() - dp.inProcessRequestIDs[deleteRequest.UserID] = deleteRequest.RequestID - dp.inProcessRequestIDsMtx.Unlock() + p.inProcessRequestIDsMtx.Lock() + p.inProcessRequests[deleteRequest.UserID] = deleteRequest + p.inProcessRequestIDsMtx.Unlock() req := makeDeleteRequestWithLogger(deleteRequest, util.Logger) level.Info(req.logger).Log("msg", "building plan for a new delete request") - err := dp.buildDeletePlan(req) + err := p.buildDeletePlan(req) if err != nil { - dp.metrics.deleteRequestsProcessingFailures.WithLabelValues(deleteRequest.UserID).Inc() + p.metrics.deleteRequestsProcessingFailures.WithLabelValues(deleteRequest.UserID).Inc() - // We do not want to remove this delete request from inProcessRequestIDs to make sure + // We do not want to remove this delete request from inProcessRequests to make sure // we do not move multiple deleting requests in deletion process. // None of the other delete requests from the user would be considered for processing until then. level.Error(req.logger).Log("msg", "error building delete plan", "err", err) @@ -421,9 +470,12 @@ func (dp *DataPurger) pullDeleteRequestsToPlanDeletes() error { } level.Info(req.logger).Log("msg", "sending delete request for execution") - dp.executePlansChan <- req + p.executePlansChan <- req } + p.metrics.oldestPendingDeleteRequestAgeSeconds.Set(float64(now.Sub(oldestPendingRequestCreatedAt) / time.Second)) + p.metrics.pendingDeleteRequestsCount.Set(float64(pendingDeleteRequestsCount)) + return nil } @@ -431,7 +483,7 @@ func (dp *DataPurger) pullDeleteRequestsToPlanDeletes() error { // A days plan will include chunk ids and labels of all the chunks which are supposed to be deleted. // Chunks are grouped together by labels to avoid storing labels repetitively. // After building delete plans it updates status of delete request to StatusDeleting and sends it for execution -func (dp *DataPurger) buildDeletePlan(req deleteRequestWithLogger) error { +func (p *Purger) buildDeletePlan(req deleteRequestWithLogger) error { ctx := context.Background() ctx = user.InjectOrgID(ctx, req.UserID) @@ -450,7 +502,7 @@ func (dp *DataPurger) buildDeletePlan(req deleteRequestWithLogger) error { return err } - chunks, err := dp.chunkStore.Get(ctx, req.UserID, planRange.Start, planRange.End, matchers...) + chunks, err := p.chunkStore.Get(ctx, req.UserID, planRange.Start, planRange.End, matchers...) if err != nil { return err } @@ -479,28 +531,28 @@ func (dp *DataPurger) buildDeletePlan(req deleteRequestWithLogger) error { plans[i] = pb } - err := dp.putDeletePlans(ctx, req.UserID, req.RequestID, plans) + err := p.putDeletePlans(ctx, req.UserID, req.RequestID, plans) if err != nil { return err } - err = dp.deleteStore.UpdateStatus(ctx, req.UserID, req.RequestID, StatusDeleting) + err = p.deleteStore.UpdateStatus(ctx, req.UserID, req.RequestID, StatusDeleting) if err != nil { return err } - dp.metrics.deleteRequestsChunksSelectedTotal.WithLabelValues(req.UserID).Add(float64(len(includedChunkIDs))) + p.metrics.deleteRequestsChunksSelectedTotal.WithLabelValues(req.UserID).Add(float64(len(includedChunkIDs))) level.Info(req.logger).Log("msg", "built delete plans", "num_plans", len(perDayTimeRange)) return nil } -func (dp *DataPurger) putDeletePlans(ctx context.Context, userID, requestID string, plans [][]byte) error { +func (p *Purger) putDeletePlans(ctx context.Context, userID, requestID string, plans [][]byte) error { for i, plan := range plans { objectKey := buildObjectKeyForPlan(userID, requestID, i) - err := dp.objectClient.PutObject(ctx, objectKey, bytes.NewReader(plan)) + err := p.objectClient.PutObject(ctx, objectKey, bytes.NewReader(plan)) if err != nil { return err } @@ -509,10 +561,10 @@ func (dp *DataPurger) putDeletePlans(ctx context.Context, userID, requestID stri return nil } -func (dp *DataPurger) getDeletePlan(ctx context.Context, userID, requestID string, planNo int) (*DeletePlan, error) { +func (p *Purger) getDeletePlan(ctx context.Context, userID, requestID string, planNo int) (*DeletePlan, error) { objectKey := buildObjectKeyForPlan(userID, requestID, planNo) - readCloser, err := dp.objectClient.GetObject(ctx, objectKey) + readCloser, err := p.objectClient.GetObject(ctx, objectKey) if err != nil { return nil, err } @@ -533,9 +585,23 @@ func (dp *DataPurger) getDeletePlan(ctx context.Context, userID, requestID strin return &plan, nil } -func (dp *DataPurger) removeDeletePlan(ctx context.Context, userID, requestID string, planNo int) error { +func (p *Purger) removeDeletePlan(ctx context.Context, userID, requestID string, planNo int) error { objectKey := buildObjectKeyForPlan(userID, requestID, planNo) - return dp.objectClient.DeleteObject(ctx, objectKey) + return p.objectClient.DeleteObject(ctx, objectKey) +} + +func (p *Purger) getOldestInProcessRequest() *DeleteRequest { + p.inProcessRequestIDsMtx.RLock() + defer p.inProcessRequestIDsMtx.RUnlock() + + var oldestRequest *DeleteRequest + for _, request := range p.inProcessRequests { + if oldestRequest == nil || request.CreatedAt.Before(oldestRequest.CreatedAt) { + oldestRequest = &request + } + } + + return oldestRequest } // returns interval per plan diff --git a/vendor/github.com/cortexproject/cortex/pkg/chunk/purger/request_handler.go b/vendor/github.com/cortexproject/cortex/pkg/chunk/purger/request_handler.go index 7b019614e967d..c8475800aee5e 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/chunk/purger/request_handler.go +++ b/vendor/github.com/cortexproject/cortex/pkg/chunk/purger/request_handler.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "net/http" + "time" "github.com/go-kit/kit/log/level" @@ -34,15 +35,17 @@ func newDeleteRequestHandlerMetrics(r prometheus.Registerer) *deleteRequestHandl // DeleteRequestHandler provides handlers for delete requests type DeleteRequestHandler struct { - deleteStore *DeleteStore - metrics *deleteRequestHandlerMetrics + deleteStore *DeleteStore + metrics *deleteRequestHandlerMetrics + deleteRequestCancelPeriod time.Duration } // NewDeleteRequestHandler creates a DeleteRequestHandler -func NewDeleteRequestHandler(deleteStore *DeleteStore, registerer prometheus.Registerer) *DeleteRequestHandler { +func NewDeleteRequestHandler(deleteStore *DeleteStore, deleteRequestCancelPeriod time.Duration, registerer prometheus.Registerer) *DeleteRequestHandler { deleteMgr := DeleteRequestHandler{ - deleteStore: deleteStore, - metrics: newDeleteRequestHandlerMetrics(registerer), + deleteStore: deleteStore, + deleteRequestCancelPeriod: deleteRequestCancelPeriod, + metrics: newDeleteRequestHandlerMetrics(registerer), } return &deleteMgr @@ -163,8 +166,8 @@ func (dm *DeleteRequestHandler) CancelDeleteRequestHandler(w http.ResponseWriter return } - if deleteRequest.CreatedAt.Add(deleteRequestCancellationDeadline).Before(model.Now()) { - http.Error(w, fmt.Sprintf("deletion of request past the deadline of %s since its creation is not allowed", deleteRequestCancellationDeadline.String()), http.StatusBadRequest) + if deleteRequest.CreatedAt.Add(dm.deleteRequestCancelPeriod).Before(model.Now()) { + http.Error(w, fmt.Sprintf("deletion of request past the deadline of %s since its creation is not allowed", dm.deleteRequestCancelPeriod.String()), http.StatusBadRequest) return } diff --git a/vendor/github.com/cortexproject/cortex/pkg/chunk/series_store.go b/vendor/github.com/cortexproject/cortex/pkg/chunk/series_store.go index 4928ca9b96455..a1e062f9770c9 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/chunk/series_store.go +++ b/vendor/github.com/cortexproject/cortex/pkg/chunk/series_store.go @@ -419,13 +419,20 @@ func (c *seriesStore) Put(ctx context.Context, chunks []Chunk) error { // PutOne implements ChunkStore func (c *seriesStore) PutOne(ctx context.Context, from, through model.Time, chunk Chunk) error { log, ctx := spanlogger.New(ctx, "SeriesStore.PutOne") - if !c.cfg.DisableChunksDeduplication { - // If this chunk is in cache it must already be in the database so we don't need to write it again - found, _, _ := c.cache.Fetch(ctx, []string{chunk.ExternalKey()}) - if len(found) > 0 { - dedupedChunksTotal.Inc() - return nil - } + writeChunk := true + + // If this chunk is in cache it must already be in the database so we don't need to write it again + found, _, _ := c.cache.Fetch(ctx, []string{chunk.ExternalKey()}) + if len(found) > 0 { + writeChunk = false + dedupedChunksTotal.Inc() + } + + // If we dont have to write the chunk and DisableIndexDeduplication is false, we do not have to do anything. + // If we dont have to write the chunk and DisableIndexDeduplication is true, we have to write index and not chunk. + // Otherwise write both index and chunk. + if !writeChunk && !c.cfg.DisableIndexDeduplication { + return nil } chunks := []Chunk{chunk} @@ -436,20 +443,31 @@ func (c *seriesStore) PutOne(ctx context.Context, from, through model.Time, chun } if oic, ok := c.storage.(ObjectAndIndexClient); ok { - if err = oic.PutChunkAndIndex(ctx, chunk, writeReqs); err != nil { + chunks := chunks + if !writeChunk { + chunks = []Chunk{} + } + if err = oic.PutChunksAndIndex(ctx, chunks, writeReqs); err != nil { return err } } else { - err := c.storage.PutChunks(ctx, chunks) - if err != nil { - return err + // chunk not found, write it. + if writeChunk { + err := c.storage.PutChunks(ctx, chunks) + if err != nil { + return err + } } if err := c.index.BatchWrite(ctx, writeReqs); err != nil { return err } } - if cacheErr := c.writeBackCache(ctx, chunks); cacheErr != nil { - level.Warn(log).Log("msg", "could not store chunks in chunk cache", "err", cacheErr) + + // we already have the chunk in the cache so don't write it back to the cache. + if writeChunk { + if cacheErr := c.writeBackCache(ctx, chunks); cacheErr != nil { + level.Warn(log).Log("msg", "could not store chunks in chunk cache", "err", cacheErr) + } } bufs := make([][]byte, len(keysToCache)) diff --git a/vendor/github.com/cortexproject/cortex/pkg/chunk/storage_client.go b/vendor/github.com/cortexproject/cortex/pkg/chunk/storage_client.go index 6797e1493f04a..84fc12d8cfb89 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/chunk/storage_client.go +++ b/vendor/github.com/cortexproject/cortex/pkg/chunk/storage_client.go @@ -40,7 +40,7 @@ type Client interface { // ObjectAndIndexClient allows optimisations where the same client handles both type ObjectAndIndexClient interface { - PutChunkAndIndex(ctx context.Context, c Chunk, index WriteBatch) error + PutChunksAndIndex(ctx context.Context, chunks []Chunk, index WriteBatch) error } // WriteBatch represents a batch of writes. diff --git a/vendor/github.com/cortexproject/cortex/pkg/compactor/blocks_cleaner.go b/vendor/github.com/cortexproject/cortex/pkg/compactor/blocks_cleaner.go index 1428176445882..411f66d336e62 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/compactor/blocks_cleaner.go +++ b/vendor/github.com/cortexproject/cortex/pkg/compactor/blocks_cleaner.go @@ -7,11 +7,13 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" + "github.com/oklog/ulid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" "github.com/thanos-io/thanos/pkg/block" + "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/compact" "github.com/thanos-io/thanos/pkg/objstore" @@ -158,7 +160,8 @@ func (c *BlocksCleaner) cleanUser(ctx context.Context, userID string) error { // Runs a bucket scan to get a fresh list of all blocks and populate // the list of deleted blocks in filter. - if _, _, err = fetcher.Fetch(ctx); err != nil { + _, partials, err := fetcher.Fetch(ctx) + if err != nil { return errors.Wrap(err, "error fetching metadata") } @@ -174,5 +177,43 @@ func (c *BlocksCleaner) cleanUser(ctx context.Context, userID string) error { return errors.Wrap(err, "error cleaning blocks") } + // Partial blocks with a deletion mark can be cleaned up. This is a best effort, so we don't return + // error if the cleanup of partial blocks fail. + if len(partials) > 0 { + level.Info(userLogger).Log("msg", "started cleaning of partial blocks marked for deletion") + c.cleanUserPartialBlocks(ctx, partials, userBucket, userLogger) + level.Info(userLogger).Log("msg", "cleaning of partial blocks marked for deletion done") + } + return nil } + +func (c *BlocksCleaner) cleanUserPartialBlocks(ctx context.Context, partials map[ulid.ULID]error, userBucket *cortex_tsdb.UserBucketClient, userLogger log.Logger) { + for blockID, blockErr := range partials { + // We can safely delete only blocks which are partial because the meta.json is missing. + if blockErr != block.ErrorSyncMetaNotFound { + continue + } + + // We can safely delete only partial blocks with a deletion mark. + _, err := metadata.ReadDeletionMark(ctx, userBucket, userLogger, blockID.String()) + if err == metadata.ErrorDeletionMarkNotFound { + continue + } + if err != nil { + level.Warn(userLogger).Log("msg", "error reading partial block deletion mark", "block", blockID, "err", err) + continue + } + + // Hard-delete partial blocks having a deletion mark, even if the deletion threshold has not + // been reached yet. + if err := block.Delete(ctx, userLogger, userBucket, blockID); err != nil { + c.blocksFailedTotal.Inc() + level.Warn(userLogger).Log("msg", "error deleting partial block marked for deletion", "block", blockID, "err", err) + continue + } + + c.blocksCleanedTotal.Inc() + level.Info(userLogger).Log("msg", "deleted partial block marked for deletion", "block", blockID) + } +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/compactor/compactor.go b/vendor/github.com/cortexproject/cortex/pkg/compactor/compactor.go index bbc3ea0032fcc..c4426b1cef850 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/compactor/compactor.go +++ b/vendor/github.com/cortexproject/cortex/pkg/compactor/compactor.go @@ -239,7 +239,7 @@ func (c *Compactor) starting(ctx context.Context) error { DataDir: c.compactorCfg.DataDir, MetaSyncConcurrency: c.compactorCfg.MetaSyncConcurrency, DeletionDelay: c.compactorCfg.DeletionDelay, - CleanupInterval: c.compactorCfg.CompactionInterval, + CleanupInterval: util.DurationWithJitter(c.compactorCfg.CompactionInterval, 0.05), }, c.bucketClient, c.usersScanner, c.parentLogger, c.registerer) // Ensure an initial cleanup occurred before starting the compactor. @@ -265,7 +265,7 @@ func (c *Compactor) running(ctx context.Context) error { // Run an initial compaction before starting the interval. c.compactUsersWithRetries(ctx) - ticker := time.NewTicker(c.compactorCfg.CompactionInterval) + ticker := time.NewTicker(util.DurationWithJitter(c.compactorCfg.CompactionInterval, 0.05)) defer ticker.Stop() for { diff --git a/vendor/github.com/cortexproject/cortex/pkg/configs/db/postgres/postgres.go b/vendor/github.com/cortexproject/cortex/pkg/configs/db/postgres/postgres.go index 6476891f62559..654e220ce5ea5 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/configs/db/postgres/postgres.go +++ b/vendor/github.com/cortexproject/cortex/pkg/configs/db/postgres/postgres.go @@ -60,7 +60,7 @@ func dbWait(db *sql.DB) error { if err == nil { return nil } - level.Warn(util.Logger).Log("msg", "db connection not established, retrying...", "error", err) + level.Warn(util.Logger).Log("msg", "db connection not established, retrying...", "err", err) time.Sleep(time.Second << uint(tries)) } return errors.Wrapf(err, "db connection not established after %s", dbTimeout) @@ -94,7 +94,7 @@ func New(uri, migrationsDir string) (DB, error) { if err != migrate.ErrNoChange { return DB{}, errors.Wrap(err, "database migrations failed") } - level.Debug(util.Logger).Log("msg", "no change in schema, error (ignored)", "error", err) + level.Debug(util.Logger).Log("msg", "no change in schema, error (ignored)", "err", err) } } @@ -340,7 +340,7 @@ func (d DB) Transaction(f func(DB) error) error { if err != nil { // Rollback error is ignored as we already have one in progress if err2 := tx.Rollback(); err2 != nil { - level.Warn(util.Logger).Log("msg", "transaction rollback error (ignored)", "error", err2) + level.Warn(util.Logger).Log("msg", "transaction rollback error (ignored)", "err", err2) } return err } diff --git a/vendor/github.com/cortexproject/cortex/pkg/configs/legacy_promql/engine.go b/vendor/github.com/cortexproject/cortex/pkg/configs/legacy_promql/engine.go index 5ff76ee4bd628..fd96474dd2467 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/configs/legacy_promql/engine.go +++ b/vendor/github.com/cortexproject/cortex/pkg/configs/legacy_promql/engine.go @@ -497,11 +497,7 @@ func (ng *Engine) populateSeries(ctx context.Context, q storage.Queryable, s *Ev params.End = params.End - offsetMilliseconds } - set, _, err = querier.Select(false, params, n.LabelMatchers...) - if err != nil { - level.Error(ng.logger).Log("msg", "error selecting series set", "err", err) - return err - } + set = querier.Select(false, params, n.LabelMatchers...) n.series, err = expandSeriesSet(ctx, set) if err != nil { // TODO(fabxc): use multi-error. @@ -520,11 +516,7 @@ func (ng *Engine) populateSeries(ctx context.Context, q storage.Queryable, s *Ev params.End = params.End - offsetMilliseconds } - set, _, err = querier.Select(false, params, n.LabelMatchers...) - if err != nil { - level.Error(ng.logger).Log("msg", "error selecting series set", "err", err) - return err - } + set = querier.Select(false, params, n.LabelMatchers...) n.series, err = expandSeriesSet(ctx, set) if err != nil { level.Error(ng.logger).Log("msg", "error expanding series set", "err", err) diff --git a/vendor/github.com/cortexproject/cortex/pkg/cortex/cortex.go b/vendor/github.com/cortexproject/cortex/pkg/cortex/cortex.go index d1389cc8a13f7..f4d276565d509 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/cortex/cortex.go +++ b/vendor/github.com/cortexproject/cortex/pkg/cortex/cortex.go @@ -12,7 +12,6 @@ import ( "github.com/go-kit/kit/log/level" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" - prom_storage "github.com/prometheus/prometheus/storage" "github.com/thanos-io/thanos/pkg/tracing" "github.com/weaveworks/common/middleware" "github.com/weaveworks/common/server" @@ -77,27 +76,27 @@ type Config struct { PrintConfig bool `yaml:"-"` HTTPPrefix string `yaml:"http_prefix"` - API api.Config `yaml:"api"` - Server server.Config `yaml:"server"` - Distributor distributor.Config `yaml:"distributor"` - Querier querier.Config `yaml:"querier"` - IngesterClient client.Config `yaml:"ingester_client"` - Ingester ingester.Config `yaml:"ingester"` - Flusher flusher.Config `yaml:"flusher"` - Storage storage.Config `yaml:"storage"` - ChunkStore chunk.StoreConfig `yaml:"chunk_store"` - Schema chunk.SchemaConfig `yaml:"schema" doc:"hidden"` // Doc generation tool doesn't support it because part of the SchemaConfig doesn't support CLI flags (needs manual documentation) - LimitsConfig validation.Limits `yaml:"limits"` - Prealloc client.PreallocConfig `yaml:"prealloc" doc:"hidden"` - Worker frontend.WorkerConfig `yaml:"frontend_worker"` - Frontend frontend.Config `yaml:"frontend"` - QueryRange queryrange.Config `yaml:"query_range"` - TableManager chunk.TableManagerConfig `yaml:"table_manager"` - Encoding encoding.Config `yaml:"-"` // No yaml for this, it only works with flags. - TSDB tsdb.Config `yaml:"tsdb"` - Compactor compactor.Config `yaml:"compactor"` - StoreGateway storegateway.Config `yaml:"store_gateway"` - DataPurgerConfig purger.Config `yaml:"purger"` + API api.Config `yaml:"api"` + Server server.Config `yaml:"server"` + Distributor distributor.Config `yaml:"distributor"` + Querier querier.Config `yaml:"querier"` + IngesterClient client.Config `yaml:"ingester_client"` + Ingester ingester.Config `yaml:"ingester"` + Flusher flusher.Config `yaml:"flusher"` + Storage storage.Config `yaml:"storage"` + ChunkStore chunk.StoreConfig `yaml:"chunk_store"` + Schema chunk.SchemaConfig `yaml:"schema" doc:"hidden"` // Doc generation tool doesn't support it because part of the SchemaConfig doesn't support CLI flags (needs manual documentation) + LimitsConfig validation.Limits `yaml:"limits"` + Prealloc client.PreallocConfig `yaml:"prealloc" doc:"hidden"` + Worker frontend.WorkerConfig `yaml:"frontend_worker"` + Frontend frontend.Config `yaml:"frontend"` + QueryRange queryrange.Config `yaml:"query_range"` + TableManager chunk.TableManagerConfig `yaml:"table_manager"` + Encoding encoding.Config `yaml:"-"` // No yaml for this, it only works with flags. + TSDB tsdb.Config `yaml:"tsdb"` + Compactor compactor.Config `yaml:"compactor"` + StoreGateway storegateway.Config `yaml:"store_gateway"` + PurgerConfig purger.Config `yaml:"purger"` Ruler ruler.Config `yaml:"ruler"` Configs configs.Config `yaml:"configs"` @@ -135,7 +134,7 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) { c.TSDB.RegisterFlags(f) c.Compactor.RegisterFlags(f) c.StoreGateway.RegisterFlags(f) - c.DataPurgerConfig.RegisterFlags(f) + c.PurgerConfig.RegisterFlags(f) c.Ruler.RegisterFlags(f) c.Configs.RegisterFlags(f) @@ -207,7 +206,7 @@ type Cortex struct { TableManager *chunk.TableManager Cache cache.Cache RuntimeConfig *runtimeconfig.Manager - DataPurger *purger.DataPurger + Purger *purger.Purger TombstonesLoader *purger.TombstonesLoader Ruler *ruler.Ruler @@ -218,9 +217,9 @@ type Cortex struct { StoreGateway *storegateway.StoreGateway MemberlistKV *memberlist.KVInitService - // Queryable that the querier should use to query the long + // Queryables that the querier should use to query the long // term storage. It depends on the storage engine used. - StoreQueryable prom_storage.Queryable + StoreQueryables []querier.QueryableWithFilter } // New makes a new Cortex. @@ -326,15 +325,15 @@ func (t *Cortex) Run() error { for m, s := range t.ServiceMap { if s == service { if service.FailureCase() == util.ErrStopProcess { - level.Info(util.Logger).Log("msg", "received stop signal via return error", "module", m, "error", service.FailureCase()) + level.Info(util.Logger).Log("msg", "received stop signal via return error", "module", m, "err", service.FailureCase()) } else { - level.Error(util.Logger).Log("msg", "module failed", "module", m, "error", service.FailureCase()) + level.Error(util.Logger).Log("msg", "module failed", "module", m, "err", service.FailureCase()) } return } } - level.Error(util.Logger).Log("msg", "module failed", "module", "unknown", "error", service.FailureCase()) + level.Error(util.Logger).Log("msg", "module failed", "module", "unknown", "err", service.FailureCase()) } sm.AddListener(services.NewManagerListener(healthy, stopped, serviceFailed)) diff --git a/vendor/github.com/cortexproject/cortex/pkg/cortex/modules.go b/vendor/github.com/cortexproject/cortex/pkg/cortex/modules.go index eac5c04b0c289..6ba55a77adb11 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/cortex/modules.go +++ b/vendor/github.com/cortexproject/cortex/pkg/cortex/modules.go @@ -3,11 +3,15 @@ package cortex import ( "fmt" "os" + "time" "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/prometheus/promql" + prom_storage "github.com/prometheus/prometheus/storage" httpgrpc_server "github.com/weaveworks/common/httpgrpc/server" + "github.com/weaveworks/common/instrument" "github.com/weaveworks/common/server" "github.com/cortexproject/cortex/pkg/alertmanager" @@ -58,7 +62,7 @@ const ( Compactor string = "compactor" StoreGateway string = "store-gateway" MemberlistKV string = "memberlist-kv" - DataPurger string = "data-purger" + Purger string = "purger" All string = "all" ) @@ -123,6 +127,11 @@ func (t *Cortex) initRuntimeConfig() (services.Service, error) { t.Cfg.RuntimeConfig.LoadPath = t.Cfg.LimitsConfig.PerTenantOverrideConfig t.Cfg.RuntimeConfig.ReloadPeriod = t.Cfg.LimitsConfig.PerTenantOverridePeriod } + + if t.Cfg.RuntimeConfig.LoadPath == "" { + // no need to initialize module if load path is empty + return nil, nil + } t.Cfg.RuntimeConfig.Loader = loadRuntimeConfig // make sure to set default limits before we start loading configuration into memory @@ -160,11 +169,19 @@ func (t *Cortex) initDistributor() (serv services.Service, err error) { } func (t *Cortex) initQuerier() (serv services.Service, err error) { - queryable, engine := querier.New(t.Cfg.Querier, t.Distributor, t.StoreQueryable, t.TombstonesLoader, prometheus.DefaultRegisterer) + queryable, engine := querier.New(t.Cfg.Querier, t.Distributor, t.StoreQueryables, t.TombstonesLoader, prometheus.DefaultRegisterer) + + // Prometheus histograms for requests to the querier. + querierRequestDuration := promauto.With(prometheus.DefaultRegisterer).NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "cortex", + Name: "querier_request_duration_seconds", + Help: "Time (in seconds) spent serving HTTP requests to the querier.", + Buckets: instrument.DefBuckets, + }, []string{"method", "route", "status_code", "ws"}) // if we are not configured for single binary mode then the querier needs to register its paths externally registerExternally := t.Cfg.Target != All - handler := t.API.RegisterQuerier(queryable, engine, t.Distributor, registerExternally, t.TombstonesLoader) + handler := t.API.RegisterQuerier(queryable, engine, t.Distributor, registerExternally, t.TombstonesLoader, querierRequestDuration) // single binary mode requires a properly configured worker. if the operator did not attempt to configure the // worker we will attempt an automatic configuration here @@ -184,37 +201,74 @@ func (t *Cortex) initQuerier() (serv services.Service, err error) { return worker, nil } -func (t *Cortex) initStoreQueryable() (services.Service, error) { - if t.Cfg.Storage.Engine == storage.StorageEngineChunks { - t.StoreQueryable = querier.NewChunkStoreQueryable(t.Cfg.Querier, t.Store) - return nil, nil +func (t *Cortex) initStoreQueryables() (services.Service, error) { + var servs []services.Service + + //nolint:golint // I prefer this form over removing 'else', because it allows q to have smaller scope. + if q, err := initQueryableForEngine(t.Cfg.Storage.Engine, t.Cfg, t.Store, prometheus.DefaultRegisterer); err != nil { + return nil, fmt.Errorf("failed to initialize querier for engine '%s': %v", t.Cfg.Storage.Engine, err) + } else { + t.StoreQueryables = append(t.StoreQueryables, querier.UseAlwaysQueryable(q)) + if s, ok := q.(services.Service); ok { + servs = append(servs, s) + } } - if t.Cfg.Storage.Engine == storage.StorageEngineTSDB && !t.Cfg.TSDB.StoreGatewayEnabled { - storeQueryable, err := querier.NewBlockQueryable(t.Cfg.TSDB, t.Cfg.Server.LogLevel, prometheus.DefaultRegisterer) + if t.Cfg.Querier.SecondStoreEngine != "" { + if t.Cfg.Querier.SecondStoreEngine == t.Cfg.Storage.Engine { + return nil, fmt.Errorf("second store engine used by querier '%s' must be different than primary engine '%s'", t.Cfg.Querier.SecondStoreEngine, t.Cfg.Storage.Engine) + } + + sq, err := initQueryableForEngine(t.Cfg.Querier.SecondStoreEngine, t.Cfg, t.Store, prometheus.DefaultRegisterer) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to initialize querier for engine '%s': %v", t.Cfg.Querier.SecondStoreEngine, err) + } + + t.StoreQueryables = append(t.StoreQueryables, querier.UseBeforeTimestampQueryable(sq, time.Time(t.Cfg.Querier.UseSecondStoreBeforeTime))) + + if s, ok := sq.(services.Service); ok { + servs = append(servs, s) } - t.StoreQueryable = storeQueryable - return storeQueryable, nil } - if t.Cfg.Storage.Engine == storage.StorageEngineTSDB && t.Cfg.TSDB.StoreGatewayEnabled { + // Return service, if any. + switch len(servs) { + case 0: + return nil, nil + case 1: + return servs[0], nil + default: + // No need to support this case yet, since chunk store is not a service. + // When we get there, we will need a wrapper service, that starts all subservices, and will also monitor them for failures. + // Not difficult, but also not necessary right now. + return nil, fmt.Errorf("too many services") + } +} + +func initQueryableForEngine(engine string, cfg Config, chunkStore chunk.Store, reg prometheus.Registerer) (prom_storage.Queryable, error) { + switch engine { + case storage.StorageEngineChunks: + if chunkStore == nil { + return nil, fmt.Errorf("chunk store not initialized") + } + return querier.NewChunkStoreQueryable(cfg.Querier, chunkStore), nil + + case storage.StorageEngineTSDB: + if !cfg.TSDB.StoreGatewayEnabled { + return querier.NewBlockQueryable(cfg.TSDB, cfg.Server.LogLevel, reg) + } + // When running in single binary, if the blocks sharding is disabled and no custom // store-gateway address has been configured, we can set it to the running process. - if t.Cfg.Target == All && !t.Cfg.StoreGateway.ShardingEnabled && t.Cfg.Querier.StoreGatewayAddresses == "" { - t.Cfg.Querier.StoreGatewayAddresses = fmt.Sprintf("127.0.0.1:%d", t.Cfg.Server.GRPCListenPort) + if cfg.Target == All && !cfg.StoreGateway.ShardingEnabled && cfg.Querier.StoreGatewayAddresses == "" { + cfg.Querier.StoreGatewayAddresses = fmt.Sprintf("127.0.0.1:%d", cfg.Server.GRPCListenPort) } - storeQueryable, err := querier.NewBlocksStoreQueryableFromConfig(t.Cfg.Querier, t.Cfg.StoreGateway, t.Cfg.TSDB, util.Logger, prometheus.DefaultRegisterer) - if err != nil { - return nil, err - } - t.StoreQueryable = storeQueryable - return storeQueryable, nil - } + return querier.NewBlocksStoreQueryableFromConfig(cfg.Querier, cfg.StoreGateway, cfg.TSDB, util.Logger, reg) - return nil, fmt.Errorf("unknown storage engine '%s'", t.Cfg.Storage.Engine) + default: + return nil, fmt.Errorf("unknown storage engine '%s'", engine) + } } func (t *Cortex) initIngester() (serv services.Service, err error) { @@ -250,8 +304,8 @@ func (t *Cortex) initFlusher() (serv services.Service, err error) { return t.Flusher, nil } -func (t *Cortex) initStore() (serv services.Service, err error) { - if t.Cfg.Storage.Engine == storage.StorageEngineTSDB { +func (t *Cortex) initChunkStore() (serv services.Service, err error) { + if t.Cfg.Storage.Engine != storage.StorageEngineChunks && t.Cfg.Querier.SecondStoreEngine != storage.StorageEngineChunks { return nil, nil } err = t.Cfg.Schema.Load() @@ -271,7 +325,7 @@ func (t *Cortex) initStore() (serv services.Service, err error) { } func (t *Cortex) initDeleteRequestsStore() (serv services.Service, err error) { - if !t.Cfg.DataPurgerConfig.Enable { + if !t.Cfg.PurgerConfig.Enable { // until we need to explicitly enable delete series support we need to do create TombstonesLoader without DeleteStore which acts as noop t.TombstonesLoader = purger.NewTombstonesLoader(nil, nil) @@ -384,7 +438,7 @@ func (t *Cortex) initTableManager() (services.Service, error) { util.CheckFatal("initializing bucket client", err) var extraTables []chunk.ExtraTables - if t.Cfg.DataPurgerConfig.Enable { + if t.Cfg.PurgerConfig.Enable { deleteStoreTableClient, err := storage.NewTableClient(t.Cfg.Storage.DeleteStoreConfig.Store, t.Cfg.Storage) if err != nil { return nil, err @@ -401,7 +455,7 @@ func (t *Cortex) initTableManager() (services.Service, error) { func (t *Cortex) initRuler() (serv services.Service, err error) { t.Cfg.Ruler.Ring.ListenPort = t.Cfg.Server.GRPCListenPort t.Cfg.Ruler.Ring.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV - queryable, engine := querier.New(t.Cfg.Querier, t.Distributor, t.StoreQueryable, t.TombstonesLoader, prometheus.DefaultRegisterer) + queryable, engine := querier.New(t.Cfg.Querier, t.Distributor, t.StoreQueryables, t.TombstonesLoader, prometheus.DefaultRegisterer) t.Ruler, err = ruler.NewRuler(t.Cfg.Ruler, engine, queryable, t.Distributor, prometheus.DefaultRegisterer, util.Logger) if err != nil { @@ -479,24 +533,24 @@ func (t *Cortex) initMemberlistKV() (services.Service, error) { return t.MemberlistKV, nil } -func (t *Cortex) initDataPurger() (services.Service, error) { - if !t.Cfg.DataPurgerConfig.Enable { +func (t *Cortex) initPurger() (services.Service, error) { + if !t.Cfg.PurgerConfig.Enable { return nil, nil } - storageClient, err := storage.NewObjectClient(t.Cfg.DataPurgerConfig.ObjectStoreType, t.Cfg.Storage) + storageClient, err := storage.NewObjectClient(t.Cfg.PurgerConfig.ObjectStoreType, t.Cfg.Storage) if err != nil { return nil, err } - t.DataPurger, err = purger.NewDataPurger(t.Cfg.DataPurgerConfig, t.DeletesStore, t.Store, storageClient, prometheus.DefaultRegisterer) + t.Purger, err = purger.NewPurger(t.Cfg.PurgerConfig, t.DeletesStore, t.Store, storageClient, prometheus.DefaultRegisterer) if err != nil { return nil, err } - t.API.RegisterPurger(t.DeletesStore) + t.API.RegisterPurger(t.DeletesStore, t.Cfg.PurgerConfig.DeleteRequestCancelPeriod) - return t.DataPurger, nil + return t.Purger, nil } func (t *Cortex) setupModuleManager() error { @@ -511,12 +565,12 @@ func (t *Cortex) setupModuleManager() error { mm.RegisterModule(Ring, t.initRing) mm.RegisterModule(Overrides, t.initOverrides) mm.RegisterModule(Distributor, t.initDistributor) - mm.RegisterModule(Store, t.initStore) + mm.RegisterModule(Store, t.initChunkStore) mm.RegisterModule(DeleteRequestsStore, t.initDeleteRequestsStore) mm.RegisterModule(Ingester, t.initIngester) mm.RegisterModule(Flusher, t.initFlusher) mm.RegisterModule(Querier, t.initQuerier) - mm.RegisterModule(StoreQueryable, t.initStoreQueryable) + mm.RegisterModule(StoreQueryable, t.initStoreQueryables) mm.RegisterModule(QueryFrontend, t.initQueryFrontend) mm.RegisterModule(TableManager, t.initTableManager) mm.RegisterModule(Ruler, t.initRuler) @@ -524,7 +578,7 @@ func (t *Cortex) setupModuleManager() error { mm.RegisterModule(AlertManager, t.initAlertManager) mm.RegisterModule(Compactor, t.initCompactor) mm.RegisterModule(StoreGateway, t.initStoreGateway) - mm.RegisterModule(DataPurger, t.initDataPurger) + mm.RegisterModule(Purger, t.initPurger) mm.RegisterModule(All, nil) mm.RegisterModule(StoreGateway, t.initStoreGateway) @@ -546,8 +600,8 @@ func (t *Cortex) setupModuleManager() error { AlertManager: {API}, Compactor: {API}, StoreGateway: {API}, - DataPurger: {Store, DeleteRequestsStore, API}, - All: {QueryFrontend, Querier, Ingester, Distributor, TableManager, DataPurger, StoreGateway}, + Purger: {Store, DeleteRequestsStore, API}, + All: {QueryFrontend, Querier, Ingester, Distributor, TableManager, Purger, StoreGateway}, } for mod, targets := range deps { if err := mm.AddDependency(mod, targets...); err != nil { diff --git a/vendor/github.com/cortexproject/cortex/pkg/cortex/runtime_config.go b/vendor/github.com/cortexproject/cortex/pkg/cortex/runtime_config.go index 8f914839720d5..44877c973c218 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/cortex/runtime_config.go +++ b/vendor/github.com/cortexproject/cortex/pkg/cortex/runtime_config.go @@ -37,6 +37,9 @@ func loadRuntimeConfig(filename string) (interface{}, error) { } func tenantLimitsFromRuntimeConfig(c *runtimeconfig.Manager) validation.TenantLimits { + if c == nil { + return nil + } return func(userID string) *validation.Limits { cfg, ok := c.GetConfig().(*runtimeConfigValues) if !ok || cfg == nil { @@ -48,6 +51,9 @@ func tenantLimitsFromRuntimeConfig(c *runtimeconfig.Manager) validation.TenantLi } func multiClientRuntimeConfigChannel(manager *runtimeconfig.Manager) func() <-chan kv.MultiRuntimeConfig { + if manager == nil { + return nil + } // returns function that can be used in MultiConfig.ConfigProvider return func() <-chan kv.MultiRuntimeConfig { outCh := make(chan kv.MultiRuntimeConfig, 1) diff --git a/vendor/github.com/cortexproject/cortex/pkg/distributor/ha_tracker.go b/vendor/github.com/cortexproject/cortex/pkg/distributor/ha_tracker.go index 5bfbd8a1eb87e..02ec53faf9c67 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/distributor/ha_tracker.go +++ b/vendor/github.com/cortexproject/cortex/pkg/distributor/ha_tracker.go @@ -236,7 +236,7 @@ func (c *haTracker) checkReplica(ctx context.Context, userID, cluster, replica s // The callback within checkKVStore will return a 202 if the sample is being deduped, // otherwise there may have been an actual error CAS'ing that we should log. if resp, ok := httpgrpc.HTTPResponseFromError(err); ok && resp.GetCode() != 202 { - level.Error(util.Logger).Log("msg", "rejecting sample", "error", err) + level.Error(util.Logger).Log("msg", "rejecting sample", "err", err) } } return err diff --git a/vendor/github.com/cortexproject/cortex/pkg/ingester/ingester_v2.go b/vendor/github.com/cortexproject/cortex/pkg/ingester/ingester_v2.go index f6cabdb686079..02a7531efbba7 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/ingester/ingester_v2.go +++ b/vendor/github.com/cortexproject/cortex/pkg/ingester/ingester_v2.go @@ -53,9 +53,7 @@ type userTSDB struct { limiter *Limiter // Thanos shipper used to ship blocks to the storage. - shipper Shipper - shipperCtx context.Context - shipperCancel context.CancelFunc + shipper Shipper // for statistics ingestedAPISamples *ewmaRate @@ -492,9 +490,9 @@ func (i *Ingester) v2Query(ctx context.Context, req *client.QueryRequest) (*clie defer q.Close() // It's not required to return sorted series because series are sorted by the Cortex querier. - ss, _, err := q.Select(false, nil, matchers...) - if err != nil { - return nil, err + ss := q.Select(false, nil, matchers...) + if ss.Err() != nil { + return nil, ss.Err() } numSamples := 0 @@ -615,9 +613,9 @@ func (i *Ingester) v2MetricsForLabelMatchers(ctx context.Context, req *client.Me } for _, matchers := range matchersSet { - seriesSet, _, err := q.Select(false, nil, matchers...) - if err != nil { - return nil, err + seriesSet := q.Select(false, nil, matchers...) + if seriesSet.Err() != nil { + return nil, seriesSet.Err() } for seriesSet.Next() { @@ -723,9 +721,9 @@ func (i *Ingester) v2QueryStream(req *client.QueryRequest, stream client.Ingeste defer q.Close() // It's not required to return sorted series because series are sorted by the Cortex querier. - ss, _, err := q.Select(false, nil, matchers...) - if err != nil { - return err + ss := q.Select(false, nil, matchers...) + if ss.Err() != nil { + return ss.Err() } timeseries := make([]client.TimeSeries, 0, queryStreamBatchSize) @@ -901,8 +899,6 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { udir, cortex_tsdb.NewUserBucketClient(userID, i.TSDBState.bucket), func() labels.Labels { return l }, metadata.ReceiveSource) - - userDB.shipperCtx, userDB.shipperCancel = context.WithCancel(context.Background()) } i.TSDBState.tsdbMetrics.setRegistryForUser(userID, tsdbPromReg) @@ -1030,20 +1026,6 @@ func (i *Ingester) numSeriesInTSDB() float64 { } func (i *Ingester) shipBlocksLoop(ctx context.Context) error { - // Start a goroutine that will cancel all shipper contexts on ingester - // shutdown, so that if there's any shipper sync in progress it will be - // quickly canceled. - // TODO: this could be a "stoppingFn" for shipper service, but let's keep that for later. - go func() { - <-ctx.Done() - - for _, userID := range i.getTSDBUsers() { - if userDB := i.getTSDB(userID); userDB != nil && userDB.shipperCancel != nil { - userDB.shipperCancel() - } - } - }() - shipTicker := time.NewTicker(i.cfg.TSDBConfig.ShipInterval) defer shipTicker.Stop() @@ -1077,13 +1059,8 @@ func (i *Ingester) shipBlocks(ctx context.Context) { return } - // Skip if the shipper context has been canceled. - if userDB.shipperCtx.Err() != nil { - return - } - // Run the shipper's Sync() to upload unshipped blocks. - if uploaded, err := userDB.shipper.Sync(userDB.shipperCtx); err != nil { + if uploaded, err := userDB.shipper.Sync(ctx); err != nil { level.Warn(util.Logger).Log("msg", "shipper failed to synchronize TSDB blocks with the storage", "user", userID, "uploaded", uploaded, "err", err) } else { level.Debug(util.Logger).Log("msg", "shipper successfully synchronized TSDB blocks with storage", "user", userID, "uploaded", uploaded) diff --git a/vendor/github.com/cortexproject/cortex/pkg/ingester/limiter.go b/vendor/github.com/cortexproject/cortex/pkg/ingester/limiter.go index 3aa111b947e45..47c37252164d1 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/ingester/limiter.go +++ b/vendor/github.com/cortexproject/cortex/pkg/ingester/limiter.go @@ -8,10 +8,10 @@ import ( ) const ( - errMaxSeriesPerMetricLimitExceeded = "per-metric series limit (local: %d global: %d actual local: %d) exceeded" - errMaxSeriesPerUserLimitExceeded = "per-user series limit (local: %d global: %d actual local: %d) exceeded" - errMaxMetadataPerMetricLimitExceeded = "per-metric metadata limit (local: %d global: %d actual local: %d) exceeded" - errMaxMetadataPerUserLimitExceeded = "per-user metric metadata limit (local: %d global %d actual local: %d) exceeded" + errMaxSeriesPerMetricLimitExceeded = "per-metric series limit (local limit: %d global limit: %d actual local limit: %d) exceeded" + errMaxSeriesPerUserLimitExceeded = "per-user series limit (local limit: %d global limit: %d actual local limit: %d) exceeded" + errMaxMetadataPerMetricLimitExceeded = "per-metric metadata limit (local limit: %d global limit: %d actual local limit: %d) exceeded" + errMaxMetadataPerUserLimitExceeded = "per-user metric metadata limit (local limit: %d global limit: %d actual local limit: %d) exceeded" ) // RingCount is the interface exposed by a ring implementation which allows diff --git a/vendor/github.com/cortexproject/cortex/pkg/ingester/wal.go b/vendor/github.com/cortexproject/cortex/pkg/ingester/wal.go index b9556a05e81e8..cf23789330eee 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/ingester/wal.go +++ b/vendor/github.com/cortexproject/cortex/pkg/ingester/wal.go @@ -8,9 +8,9 @@ import ( "math" "os" "path/filepath" + "regexp" "runtime" "strconv" - "strings" "sync" "time" @@ -413,16 +413,13 @@ func lastCheckpoint(dir string) (string, int, error) { for i := 0; i < len(dirs); i++ { di := dirs[i] - if !strings.HasPrefix(di.Name(), checkpointPrefix) { + idx, err := checkpointIndex(di.Name(), false) + if err != nil { continue } if !di.IsDir() { return "", -1, fmt.Errorf("checkpoint %s is not a directory", di.Name()) } - idx, err := strconv.Atoi(di.Name()[len(checkpointPrefix):]) - if err != nil { - continue - } if idx > maxIdx { checkpointDir = di.Name() maxIdx = idx @@ -450,10 +447,7 @@ func (w *walWrapper) deleteCheckpoints(maxIndex int) (err error) { return err } for _, fi := range files { - if !strings.HasPrefix(fi.Name(), checkpointPrefix) { - continue - } - index, err := strconv.Atoi(fi.Name()[len(checkpointPrefix):]) + index, err := checkpointIndex(fi.Name(), true) if err != nil || index >= maxIndex { continue } @@ -464,6 +458,23 @@ func (w *walWrapper) deleteCheckpoints(maxIndex int) (err error) { return errs.Err() } +var checkpointRe = regexp.MustCompile("^" + regexp.QuoteMeta(checkpointPrefix) + "(\\d+)(\\.tmp)?$") + +// checkpointIndex returns the index of a given checkpoint file. It handles +// both regular and temporary checkpoints according to the includeTmp flag. If +// the file is not a checkpoint it returns an error. +func checkpointIndex(filename string, includeTmp bool) (int, error) { + result := checkpointRe.FindStringSubmatch(filename) + if len(result) < 2 { + return 0, errors.New("file is not a checkpoint") + } + // Filter out temporary checkpoints if desired. + if !includeTmp && len(result) == 3 && result[2] != "" { + return 0, errors.New("temporary checkpoint") + } + return strconv.Atoi(result[1]) +} + // checkpointSeries write the chunks of the series to the checkpoint. func (w *walWrapper) checkpointSeries(userID string, fp model.Fingerprint, series *memorySeries, wireChunks []client.Chunk, b []byte) ([]client.Chunk, []byte, error) { var err error diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/block.go b/vendor/github.com/cortexproject/cortex/pkg/querier/block.go index 95cd61a3d8bc8..f3718f7437916 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/block.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/block.go @@ -86,7 +86,7 @@ type blocksQuerier struct { // Select implements storage.Querier interface. // The bool passed is ignored because the series is always sorted. -func (b *blocksQuerier) Select(_ bool, sp *storage.SelectHints, matchers ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (b *blocksQuerier) Select(_ bool, sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { log, ctx := spanlogger.New(b.ctx, "blocksQuerier.Select") defer log.Span.Finish() @@ -106,12 +106,15 @@ func (b *blocksQuerier) Select(_ bool, sp *storage.SelectHints, matchers ...*lab PartialResponseStrategy: storepb.PartialResponseStrategy_ABORT, }) if err != nil { - return nil, nil, promql.ErrStorage{Err: err} + return storage.ErrSeriesSet(promql.ErrStorage{Err: err}) } level.Debug(log).Log("series", len(series), "warnings", len(warnings)) - return &blockQuerierSeriesSet{series: series}, warnings, nil + return &blockQuerierSeriesSet{ + series: series, + warnings: warnings, + } } func convertMatchersToLabelMatcher(matchers []*labels.Matcher) []storepb.LabelMatcher { @@ -155,7 +158,8 @@ func (b *blocksQuerier) Close() error { // Implementation of storage.SeriesSet, based on individual responses from store client. type blockQuerierSeriesSet struct { - series []*storepb.Series + series []*storepb.Series + warnings storage.Warnings // next response to process next int @@ -195,6 +199,10 @@ func (bqss *blockQuerierSeriesSet) Err() error { return nil } +func (bqss *blockQuerierSeriesSet) Warnings() storage.Warnings { + return bqss.warnings +} + // newBlockQuerierSeries makes a new blockQuerierSeries. Input labels must be already sorted by name. func newBlockQuerierSeries(lbls []storepb.Label, chunks []storepb.AggrChunk) *blockQuerierSeries { sort.Slice(chunks, func(i, j int) bool { diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_consistency_checker.go b/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_consistency_checker.go index 9bfb4b2b61b7c..6edebf057cd12 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_consistency_checker.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_consistency_checker.go @@ -1,8 +1,6 @@ package querier import ( - "fmt" - "strings" "time" "github.com/go-kit/kit/log" @@ -11,7 +9,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/thanos-io/thanos/pkg/block/metadata" - "github.com/thanos-io/thanos/pkg/store/hintspb" ) type BlocksConsistencyChecker struct { @@ -39,24 +36,17 @@ func NewBlocksConsistencyChecker(uploadGracePeriod, deletionGracePeriod time.Dur } } -func (c *BlocksConsistencyChecker) Check(expectedBlocks []*BlockMeta, knownDeletionMarks map[ulid.ULID]*metadata.DeletionMark, queriedBlocks map[string][]hintspb.Block) error { +func (c *BlocksConsistencyChecker) Check(knownBlocks []*BlockMeta, knownDeletionMarks map[ulid.ULID]*metadata.DeletionMark, queriedBlocks []ulid.ULID) (missingBlocks []ulid.ULID) { c.checksTotal.Inc() - // Reverse the map of queried blocks, so that we can easily look for missing ones - // while keeping the information about which store-gateways have already been queried - // for that block. - actualBlocks := map[string][]string{} - for gatewayAddr, blocks := range queriedBlocks { - for _, b := range blocks { - actualBlocks[b.Id] = append(actualBlocks[b.Id], gatewayAddr) - } + // Reverse the map of queried blocks, so that we can easily look for missing ones. + actualBlocks := map[ulid.ULID]struct{}{} + for _, blockID := range queriedBlocks { + actualBlocks[blockID] = struct{}{} } // Look for any missing block. - missingBlocks := map[string][]string{} - var missingBlockIDs []string - - for _, meta := range expectedBlocks { + for _, meta := range knownBlocks { // Some recently uploaded blocks, already discovered by the querier, may not have been discovered // and loaded by the store-gateway yet. In order to avoid false positives, we grant some time // to the store-gateway to discover them. It's safe to exclude recently uploaded blocks because: @@ -82,17 +72,14 @@ func (c *BlocksConsistencyChecker) Check(expectedBlocks []*BlockMeta, knownDelet } } - id := meta.ULID.String() - if gatewayAddrs, ok := actualBlocks[id]; !ok { - missingBlocks[id] = gatewayAddrs - missingBlockIDs = append(missingBlockIDs, id) + if _, ok := actualBlocks[meta.ULID]; !ok { + missingBlocks = append(missingBlocks, meta.ULID) } } - if len(missingBlocks) == 0 { - return nil + if len(missingBlocks) > 0 { + c.checksFailed.Inc() } - c.checksFailed.Inc() - return fmt.Errorf("consistency check failed because some blocks were not queried: %s", strings.Join(missingBlockIDs, " ")) + return missingBlocks } diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_store_balanced_set.go b/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_store_balanced_set.go index 61381a7609ccd..e81266ca1b9b3 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_store_balanced_set.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_store_balanced_set.go @@ -58,21 +58,44 @@ func (s *blocksStoreBalancedSet) resolve(ctx context.Context) error { return nil } -func (s *blocksStoreBalancedSet) GetClientsFor(blockIDs []ulid.ULID) (map[BlocksStoreClient][]ulid.ULID, error) { +func (s *blocksStoreBalancedSet) GetClientsFor(blockIDs []ulid.ULID, exclude map[ulid.ULID][]string) (map[BlocksStoreClient][]ulid.ULID, error) { addresses := s.dnsProvider.Addresses() if len(addresses) == 0 { return nil, fmt.Errorf("no address resolved for the store-gateway service addresses %s", strings.Join(s.serviceAddresses, ",")) } - // Pick a random address and return its client from the pool. - addr := addresses[rand.Intn(len(addresses))] - c, err := s.clientsPool.GetClientFor(addr) - if err != nil { - return nil, errors.Wrapf(err, "failed to get store-gateway client for %s", addr) - } + // Randomize the list of addresses to not always query the same address. + rand.Shuffle(len(addresses), func(i, j int) { + addresses[i], addresses[j] = addresses[j], addresses[i] + }) + // Pick a non excluded client for each block. clients := map[BlocksStoreClient][]ulid.ULID{} - clients[c.(BlocksStoreClient)] = blockIDs + + for _, blockID := range blockIDs { + // Pick the first non excluded store-gateway instance. + addr := getFirstNonExcludedAddr(addresses, exclude[blockID]) + if addr == "" { + return nil, fmt.Errorf("no store-gateway instance left after filtering out excluded instances for block %s", blockID.String()) + } + + c, err := s.clientsPool.GetClientFor(addr) + if err != nil { + return nil, errors.Wrapf(err, "failed to get store-gateway client for %s", addr) + } + + clients[c.(BlocksStoreClient)] = append(clients[c.(BlocksStoreClient)], blockID) + } return clients, nil } + +func getFirstNonExcludedAddr(addresses, exclude []string) string { + for _, addr := range addresses { + if !util.StringsContain(exclude, addr) { + return addr + } + } + + return "" +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_store_queryable.go b/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_store_queryable.go index db1008c3b643c..c9f28c3e6e5e7 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_store_queryable.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_store_queryable.go @@ -2,7 +2,7 @@ package querier import ( "context" - "flag" + "fmt" "io" "strings" "sync" @@ -38,6 +38,13 @@ import ( "github.com/cortexproject/cortex/pkg/util/spanlogger" ) +const ( + // The maximum number of times we attempt fetching missing blocks from different + // store-gateways. If no more store-gateways are left (ie. due to lower replication + // factor) than we'll end the retries earlier. + maxFetchSeriesAttempts = 3 +) + var ( errNoStoreGatewayAddress = errors.New("no store-gateway address configured") ) @@ -47,8 +54,9 @@ type BlocksStoreSet interface { services.Service // GetClientsFor returns the store gateway clients that should be used to - // query the set of blocks in input. - GetClientsFor(blockIDs []ulid.ULID) (map[BlocksStoreClient][]ulid.ULID, error) + // query the set of blocks in input. The exclude parameter is the map of + // blocks -> store-gateway addresses that should be excluded. + GetClientsFor(blockIDs []ulid.ULID, exclude map[ulid.ULID][]string) (map[BlocksStoreClient][]ulid.ULID, error) } // BlocksFinder is the interface used to find blocks for a given user and time range. @@ -70,12 +78,26 @@ type BlocksStoreClient interface { RemoteAddress() string } -type BlocksConsistencyCheckConfig struct { - Enabled bool `yaml:"enabled"` +type blocksStoreQueryableMetrics struct { + storesHit prometheus.Histogram + refetches prometheus.Histogram } -func (cfg *BlocksConsistencyCheckConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { - f.BoolVar(&cfg.Enabled, prefix+".enabled", false, "Whether the querier should run a consistency check to ensure all expected blocks have been queried.") +func newBlocksStoreQueryableMetrics(reg prometheus.Registerer) *blocksStoreQueryableMetrics { + return &blocksStoreQueryableMetrics{ + storesHit: promauto.With(reg).NewHistogram(prometheus.HistogramOpts{ + Namespace: "cortex", + Name: "querier_storegateway_instances_hit_per_query", + Help: "Number of store-gateway instances hit for a single query.", + Buckets: []float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + }), + refetches: promauto.With(reg).NewHistogram(prometheus.HistogramOpts{ + Namespace: "cortex", + Name: "querier_storegateway_refetches_per_query", + Help: "Number of re-fetches attempted while querying store-gateway instances due to missing blocks.", + Buckets: []float64{0, 1, 2}, + }), + } } // BlocksStoreQueryable is a queryable which queries blocks storage via @@ -88,13 +110,11 @@ type BlocksStoreQueryable struct { consistency *BlocksConsistencyChecker logger log.Logger queryStoreAfter time.Duration + metrics *blocksStoreQueryableMetrics // Subservices manager. subservices *services.Manager subservicesWatcher *services.FailureWatcher - - // Metrics. - storesHit prometheus.Histogram } func NewBlocksStoreQueryable(stores BlocksStoreSet, finder BlocksFinder, consistency *BlocksConsistencyChecker, queryStoreAfter time.Duration, logger log.Logger, reg prometheus.Registerer) (*BlocksStoreQueryable, error) { @@ -113,12 +133,7 @@ func NewBlocksStoreQueryable(stores BlocksStoreSet, finder BlocksFinder, consist logger: logger, subservices: manager, subservicesWatcher: services.NewFailureWatcher(), - storesHit: promauto.With(reg).NewHistogram(prometheus.HistogramOpts{ - Namespace: "cortex", - Name: "querier_storegateway_instances_hit_per_query", - Help: "Number of store-gateway instances hit for a single query.", - Buckets: []float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, - }), + metrics: newBlocksStoreQueryableMetrics(reg), } q.Service = services.NewBasicService(q.starting, q.running, q.stopping) @@ -182,20 +197,17 @@ func NewBlocksStoreQueryableFromConfig(querierCfg Config, gatewayCfg storegatewa stores = newBlocksStoreBalancedSet(querierCfg.GetStoreGatewayAddresses(), querierCfg.StoreGatewayClient, logger, reg) } - var consistency *BlocksConsistencyChecker - if querierCfg.BlocksConsistencyCheck.Enabled { - consistency = NewBlocksConsistencyChecker( - // Exclude blocks which have been recently uploaded, in order to give enough time to store-gateways - // to discover and load them (3 times the sync interval). - storageCfg.BucketStore.ConsistencyDelay+(3*storageCfg.BucketStore.SyncInterval), - // To avoid any false positive in the consistency check, we do exclude blocks which have been - // recently marked for deletion, until the "ignore delay / 2". This means the consistency checker - // exclude such blocks about 50% of the time before querier and store-gateway stops querying them. - storageCfg.BucketStore.IgnoreDeletionMarksDelay/2, - logger, - reg, - ) - } + consistency := NewBlocksConsistencyChecker( + // Exclude blocks which have been recently uploaded, in order to give enough time to store-gateways + // to discover and load them (3 times the sync interval). + storageCfg.BucketStore.ConsistencyDelay+(3*storageCfg.BucketStore.SyncInterval), + // To avoid any false positive in the consistency check, we do exclude blocks which have been + // recently marked for deletion, until the "ignore delay / 2". This means the consistency checker + // exclude such blocks about 50% of the time before querier and store-gateway stops querying them. + storageCfg.BucketStore.IgnoreDeletionMarksDelay/2, + logger, + reg, + ) return NewBlocksStoreQueryable(stores, scanner, consistency, querierCfg.QueryStoreAfter, logger, reg) } @@ -243,7 +255,7 @@ func (q *BlocksStoreQueryable) Querier(ctx context.Context, mint, maxt int64) (s userID: userID, finder: q.finder, stores: q.stores, - storesHit: q.storesHit, + metrics: q.metrics, consistency: q.consistency, logger: q.logger, queryStoreAfter: q.queryStoreAfter, @@ -256,7 +268,7 @@ type blocksStoreQuerier struct { userID string finder BlocksFinder stores BlocksStoreSet - storesHit prometheus.Histogram + metrics *blocksStoreQueryableMetrics consistency *BlocksConsistencyChecker logger log.Logger @@ -267,15 +279,15 @@ type blocksStoreQuerier struct { // Select implements storage.Querier interface. // The bool passed is ignored because the series is always sorted. -func (q *blocksStoreQuerier) Select(_ bool, sp *storage.SelectHints, matchers ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { - set, warnings, err := q.selectSorted(sp, matchers...) +func (q *blocksStoreQuerier) Select(_ bool, sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { + set := q.selectSorted(sp, matchers...) // We need to wrap the error in order to have Prometheus returning a 5xx error. - if err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { - err = promql.ErrStorage{Err: err} + if err := set.Err(); err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { + set = storage.ErrSeriesSet(promql.ErrStorage{Err: err}) } - return set, warnings, err + return set } func (q *blocksStoreQuerier) LabelValues(name string) ([]string, storage.Warnings, error) { @@ -292,7 +304,7 @@ func (q *blocksStoreQuerier) Close() error { return nil } -func (q *blocksStoreQuerier) selectSorted(sp *storage.SelectHints, matchers ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (q *blocksStoreQuerier) selectSorted(sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { spanLog, spanCtx := spanlogger.New(q.ctx, "blocksStoreQuerier.selectSorted") defer spanLog.Span.Finish() @@ -315,42 +327,114 @@ func (q *blocksStoreQuerier) selectSorted(sp *storage.SelectHints, matchers ...* } if maxT < minT { - q.storesHit.Observe(0) + q.metrics.storesHit.Observe(0) level.Debug(spanLog).Log("msg", "empty query time range after max time manipulation") - return series.NewEmptySeriesSet(), nil, nil + return storage.EmptySeriesSet() } } // Find the list of blocks we need to query given the time range. - metas, deletionMarks, err := q.finder.GetBlocks(q.userID, minT, maxT) + knownMetas, knownDeletionMarks, err := q.finder.GetBlocks(q.userID, minT, maxT) if err != nil { - return nil, nil, err + return storage.ErrSeriesSet(err) } - if len(metas) == 0 { - q.storesHit.Observe(0) + if len(knownMetas) == 0 { + q.metrics.storesHit.Observe(0) level.Debug(spanLog).Log("msg", "no blocks found") - return series.NewEmptySeriesSet(), nil, nil + return storage.EmptySeriesSet() } - level.Debug(spanLog).Log("msg", "found blocks to query", "expected", BlockMetas(metas).String()) + level.Debug(spanLog).Log("msg", "found blocks to query", "expected", BlockMetas(knownMetas).String()) - // Find the set of store-gateway instances having the blocks. - blockIDs := getULIDsFromBlockMetas(metas) - clients, err := q.stores.GetClientsFor(blockIDs) - if err != nil { - return nil, nil, err + var ( + // At the beginning the list of blocks to query are all known blocks. + remainingBlocks = getULIDsFromBlockMetas(knownMetas) + attemptedBlocks = map[ulid.ULID][]string{} + touchedStores = map[string]struct{}{} + + convertedMatchers = convertMatchersToLabelMatcher(matchers) + resSeriesSets = []storage.SeriesSet(nil) + resWarnings = storage.Warnings(nil) + resQueriedBlocks = []ulid.ULID(nil) + ) + + for attempt := 1; attempt <= maxFetchSeriesAttempts; attempt++ { + // Find the set of store-gateway instances having the blocks. The exclude parameter is the + // map of blocks queried so far, with the list of store-gateway addresses for each block. + clients, err := q.stores.GetClientsFor(remainingBlocks, attemptedBlocks) + if err != nil { + // If it's a retry and we get an error, it means there are no more store-gateways left + // from which running another attempt, so we're just stopping retrying. + if attempt > 1 { + level.Warn(spanLog).Log("msg", "unable to get store-gateway clients while retrying to fetch missing blocks", "err", err) + break + } + + return storage.ErrSeriesSet(err) + } + level.Debug(spanLog).Log("msg", "found store-gateway instances to query", "num instances", len(clients), "attempt", attempt) + + // Fetch series from stores. If an error occur we do not retry because retries + // are only meant to cover missing blocks. + seriesSets, queriedBlocks, warnings, err := q.fetchSeriesFromStores(spanCtx, clients, minT, maxT, convertedMatchers) + if err != nil { + return storage.ErrSeriesSet(err) + } + level.Debug(spanLog).Log("msg", "received series from all store-gateways", "queried blocks", strings.Join(convertULIDsToString(queriedBlocks), " ")) + + resSeriesSets = append(resSeriesSets, seriesSets...) + resWarnings = append(resWarnings, warnings...) + resQueriedBlocks = append(resQueriedBlocks, queriedBlocks...) + + // Update the map of blocks we attempted to query. + for client, blockIDs := range clients { + touchedStores[client.RemoteAddress()] = struct{}{} + + for _, blockID := range blockIDs { + attemptedBlocks[blockID] = append(attemptedBlocks[blockID], client.RemoteAddress()) + } + } + + // Ensure all expected blocks have been queried (during all tries done so far). + missingBlocks := q.consistency.Check(knownMetas, knownDeletionMarks, resQueriedBlocks) + if len(missingBlocks) == 0 { + q.metrics.storesHit.Observe(float64(len(touchedStores))) + q.metrics.refetches.Observe(float64(attempt - 1)) + + return series.NewSeriesSetWithWarnings( + storage.NewMergeSeriesSet(resSeriesSets, storage.ChainedSeriesMerge), + resWarnings) + } + + level.Debug(spanLog).Log("msg", "consistency check failed", "attempt", attempt, "missing blocks", strings.Join(convertULIDsToString(missingBlocks), " ")) + + // The next attempt should just query the missing blocks. + remainingBlocks = missingBlocks } - level.Debug(spanLog).Log("msg", "found store-gateway instances to query", "num instances", len(clients)) + // We've not been able to query all expected blocks after all retries. + err = fmt.Errorf("consistency check failed because some blocks were not queried: %s", strings.Join(convertULIDsToString(remainingBlocks), " ")) + level.Warn(util.WithContext(spanCtx, spanLog)).Log("msg", "failed consistency check", "err", err) + + return storage.ErrSeriesSet(err) +} + +func (q *blocksStoreQuerier) fetchSeriesFromStores( + ctx context.Context, + clients map[BlocksStoreClient][]ulid.ULID, + minT int64, + maxT int64, + matchers []storepb.LabelMatcher, +) ([]storage.SeriesSet, []ulid.ULID, storage.Warnings, error) { var ( - reqCtx = grpc_metadata.AppendToOutgoingContext(spanCtx, cortex_tsdb.TenantIDExternalLabel, q.userID) - g, gCtx = errgroup.WithContext(reqCtx) - mtx = sync.Mutex{} - seriesSets = []storage.SeriesSet(nil) - warnings = storage.Warnings(nil) - queriedBlocks = map[string][]hintspb.Block{} - convertedMatchers = convertMatchersToLabelMatcher(matchers) + reqCtx = grpc_metadata.AppendToOutgoingContext(ctx, cortex_tsdb.TenantIDExternalLabel, q.userID) + g, gCtx = errgroup.WithContext(reqCtx) + mtx = sync.Mutex{} + seriesSets = []storage.SeriesSet(nil) + warnings = storage.Warnings(nil) + queriedBlocks = []ulid.ULID(nil) + spanLog = spanlogger.FromContext(ctx) ) // Concurrently fetch series from all clients. @@ -360,7 +444,7 @@ func (q *blocksStoreQuerier) selectSorted(sp *storage.SelectHints, matchers ...* blockIDs := blockIDs g.Go(func() error { - req, err := createSeriesRequest(minT, maxT, convertedMatchers, blockIDs) + req, err := createSeriesRequest(minT, maxT, matchers, blockIDs) if err != nil { return errors.Wrapf(err, "failed to create series request") } @@ -372,7 +456,7 @@ func (q *blocksStoreQuerier) selectSorted(sp *storage.SelectHints, matchers ...* mySeries := []*storepb.Series(nil) myWarnings := storage.Warnings(nil) - myQueriedBlocks := []hintspb.Block(nil) + myQueriedBlocks := []ulid.ULID(nil) for { resp, err := stream.Recv() @@ -397,7 +481,13 @@ func (q *blocksStoreQuerier) selectSorted(sp *storage.SelectHints, matchers ...* if err := types.UnmarshalAny(h, &hints); err != nil { return errors.Wrapf(err, "failed to unmarshal hints from %s", c) } - myQueriedBlocks = append(myQueriedBlocks, hints.QueriedBlocks...) + + ids, err := convertBlockHintsToULIDs(hints.QueriedBlocks) + if err != nil { + return errors.Wrapf(err, "failed to parse queried block IDs from received hints") + } + + myQueriedBlocks = append(myQueriedBlocks, ids...) } } @@ -405,14 +495,14 @@ func (q *blocksStoreQuerier) selectSorted(sp *storage.SelectHints, matchers ...* "instance", c, "num series", len(mySeries), "bytes series", countSeriesBytes(mySeries), - "requested blocks", strings.Join(convertULIDsToString(blockIDs), ","), - "queried blocks", strings.Join(convertBlockHintsToString(myQueriedBlocks), ",")) + "requested blocks", strings.Join(convertULIDsToString(blockIDs), " "), + "queried blocks", strings.Join(convertULIDsToString(myQueriedBlocks), " ")) // Store the result. mtx.Lock() seriesSets = append(seriesSets, &blockQuerierSeriesSet{series: mySeries}) warnings = append(warnings, myWarnings...) - queriedBlocks[c.RemoteAddress()] = myQueriedBlocks + queriedBlocks = append(queriedBlocks, myQueriedBlocks...) mtx.Unlock() return nil @@ -421,21 +511,10 @@ func (q *blocksStoreQuerier) selectSorted(sp *storage.SelectHints, matchers ...* // Wait until all client requests complete. if err := g.Wait(); err != nil { - return nil, nil, err + return nil, nil, nil, err } - level.Debug(spanLog).Log("msg", "received series from all store-gateways", "queried blocks", queriedBlocks) - q.storesHit.Observe(float64(len(clients))) - - // Ensure all expected blocks have been queried. - if q.consistency != nil { - if err := q.consistency.Check(metas, deletionMarks, queriedBlocks); err != nil { - level.Warn(util.WithContext(q.ctx, q.logger)).Log("msg", "failed consistency check", "err", err) - return nil, nil, err - } - } - - return storage.NewMergeSeriesSet(seriesSets, storage.ChainedSeriesMerge), warnings, nil + return seriesSets, queriedBlocks, warnings, nil } func createSeriesRequest(minT, maxT int64, matchers []storepb.LabelMatcher, blockIDs []ulid.ULID) (*storepb.SeriesRequest, error) { @@ -472,12 +551,19 @@ func convertULIDsToString(ids []ulid.ULID) []string { return res } -func convertBlockHintsToString(hints []hintspb.Block) []string { - res := make([]string, len(hints)) +func convertBlockHintsToULIDs(hints []hintspb.Block) ([]ulid.ULID, error) { + res := make([]ulid.ULID, len(hints)) + for idx, hint := range hints { - res[idx] = hint.Id + blockID, err := ulid.Parse(hint.Id) + if err != nil { + return nil, err + } + + res[idx] = blockID } - return res + + return res, nil } func countSeriesBytes(series []*storepb.Series) (count uint64) { diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_store_replicated_set.go b/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_store_replicated_set.go index cc6d1cd2998e0..8c9beb607a242 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_store_replicated_set.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_store_replicated_set.go @@ -2,6 +2,7 @@ package querier import ( "context" + "fmt" "github.com/go-kit/kit/log" "github.com/oklog/ulid" @@ -11,6 +12,7 @@ import ( "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/ring/client" cortex_tsdb "github.com/cortexproject/cortex/pkg/storage/tsdb" + "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/services" "github.com/cortexproject/cortex/pkg/util/tls" ) @@ -70,7 +72,7 @@ func (s *blocksStoreReplicationSet) stopping(_ error) error { return services.StopManagerAndAwaitStopped(context.Background(), s.subservices) } -func (s *blocksStoreReplicationSet) GetClientsFor(blockIDs []ulid.ULID) (map[BlocksStoreClient][]ulid.ULID, error) { +func (s *blocksStoreReplicationSet) GetClientsFor(blockIDs []ulid.ULID, exclude map[ulid.ULID][]string) (map[BlocksStoreClient][]ulid.ULID, error) { shards := map[string][]ulid.ULID{} // Find the replication set of each block we need to query. @@ -85,8 +87,11 @@ func (s *blocksStoreReplicationSet) GetClientsFor(blockIDs []ulid.ULID) (map[Blo return nil, errors.Wrapf(err, "failed to get store-gateway replication set owning the block %s", blockID.String()) } - // Pick the first store-gateway instance. - addr := set.Ingesters[0].Addr + // Pick the first non excluded store-gateway instance. + addr := getFirstNonExcludedInstanceAddr(set, exclude[blockID]) + if addr == "" { + return nil, fmt.Errorf("no store-gateway instance left after checking exclude for block %s", blockID.String()) + } shards[addr] = append(shards[addr], blockID) } @@ -105,3 +110,13 @@ func (s *blocksStoreReplicationSet) GetClientsFor(blockIDs []ulid.ULID) (map[Blo return clients, nil } + +func getFirstNonExcludedInstanceAddr(set ring.ReplicationSet, exclude []string) string { + for _, instance := range set.Ingesters { + if !util.StringsContain(exclude, instance.Addr) { + return instance.Addr + } + } + + return "" +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/chunk_store_queryable.go b/vendor/github.com/cortexproject/cortex/pkg/querier/chunk_store_queryable.go index 162ab736b29f3..50fa8573fb642 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/chunk_store_queryable.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/chunk_store_queryable.go @@ -39,10 +39,10 @@ type chunkStoreQuerier struct { // Select implements storage.Querier interface. // The bool passed is ignored because the series is always sorted. -func (q *chunkStoreQuerier) Select(_ bool, sp *storage.SelectHints, matchers ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (q *chunkStoreQuerier) Select(_ bool, sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { userID, err := user.ExtractOrgID(q.ctx) if err != nil { - return nil, nil, err + return storage.ErrSeriesSet(err) } chunks, err := q.store.Get(q.ctx, userID, model.Time(sp.Start), model.Time(sp.End), matchers...) if err != nil { @@ -50,18 +50,18 @@ func (q *chunkStoreQuerier) Select(_ bool, sp *storage.SelectHints, matchers ... case promql.ErrStorage, promql.ErrTooManySamples, promql.ErrQueryCanceled, promql.ErrQueryTimeout: // Recognized by Prometheus API, vendor/github.com/prometheus/prometheus/promql/engine.go:91. // Don't translate those, just in case we use them internally. - return nil, nil, err + return storage.ErrSeriesSet(err) case chunk.QueryError: // This will be returned with status code 422 by Prometheus API. // vendor/github.com/prometheus/prometheus/web/api/v1/api.go:1393 - return nil, nil, err + return storage.ErrSeriesSet(err) default: // All other errors will be returned as 500. - return nil, nil, promql.ErrStorage{Err: err} + return storage.ErrSeriesSet(promql.ErrStorage{Err: err}) } } - return partitionChunks(chunks, q.mint, q.maxt, q.chunkIteratorFunc), nil, nil + return partitionChunks(chunks, q.mint, q.maxt, q.chunkIteratorFunc) } // Series in the returned set are sorted alphabetically by labels. diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/distributor_queryable.go b/vendor/github.com/cortexproject/cortex/pkg/querier/distributor_queryable.go index b9d74301bd680..675946f0e4ccf 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/distributor_queryable.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/distributor_queryable.go @@ -3,6 +3,7 @@ package querier import ( "context" "sort" + "time" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/pkg/labels" @@ -14,6 +15,7 @@ import ( "github.com/cortexproject/cortex/pkg/ingester/client" "github.com/cortexproject/cortex/pkg/prom1/storage/metric" "github.com/cortexproject/cortex/pkg/querier/series" + "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/chunkcompat" "github.com/cortexproject/cortex/pkg/util/spanlogger" ) @@ -29,17 +31,36 @@ type Distributor interface { MetricsMetadata(ctx context.Context) ([]scrape.MetricMetadata, error) } -func newDistributorQueryable(distributor Distributor, streaming bool, iteratorFn chunkIteratorFunc) storage.Queryable { - return storage.QueryableFunc(func(ctx context.Context, mint, maxt int64) (storage.Querier, error) { - return &distributorQuerier{ - distributor: distributor, - ctx: ctx, - mint: mint, - maxt: maxt, - streaming: streaming, - chunkIterFn: iteratorFn, - }, nil - }) +func newDistributorQueryable(distributor Distributor, streaming bool, iteratorFn chunkIteratorFunc, queryIngesterWithin time.Duration) QueryableWithFilter { + return distributorQueryable{ + distributor: distributor, + streaming: streaming, + iteratorFn: iteratorFn, + queryIngesterWithin: queryIngesterWithin, + } +} + +type distributorQueryable struct { + distributor Distributor + streaming bool + iteratorFn chunkIteratorFunc + queryIngesterWithin time.Duration +} + +func (d distributorQueryable) Querier(ctx context.Context, mint, maxt int64) (storage.Querier, error) { + return &distributorQuerier{ + distributor: d.distributor, + ctx: ctx, + mint: mint, + maxt: maxt, + streaming: d.streaming, + chunkIterFn: d.iteratorFn, + }, nil +} + +func (d distributorQueryable) UseQueryable(now time.Time, _, queryMaxT int64) bool { + // Include ingester only if maxt is within QueryIngestersWithin w.r.t. current time. + return d.queryIngesterWithin == 0 || queryMaxT >= util.TimeToMillis(now.Add(-d.queryIngesterWithin)) } type distributorQuerier struct { @@ -52,7 +73,7 @@ type distributorQuerier struct { // Select implements storage.Querier interface. // The bool passed is ignored because the series is always sorted. -func (q *distributorQuerier) Select(_ bool, sp *storage.SelectHints, matchers ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (q *distributorQuerier) Select(_ bool, sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { log, ctx := spanlogger.New(q.ctx, "distributorQuerier.Select") defer log.Span.Finish() @@ -61,9 +82,9 @@ func (q *distributorQuerier) Select(_ bool, sp *storage.SelectHints, matchers .. if sp == nil { ms, err := q.distributor.MetricsForLabelMatchers(ctx, model.Time(q.mint), model.Time(q.maxt), matchers...) if err != nil { - return nil, nil, err + return storage.ErrSeriesSet(err) } - return series.MetricsToSeriesSet(ms), nil, nil + return series.MetricsToSeriesSet(ms) } mint, maxt := sp.Start, sp.End @@ -74,28 +95,28 @@ func (q *distributorQuerier) Select(_ bool, sp *storage.SelectHints, matchers .. matrix, err := q.distributor.Query(ctx, model.Time(mint), model.Time(maxt), matchers...) if err != nil { - return nil, nil, promql.ErrStorage{Err: err} + return storage.ErrSeriesSet(promql.ErrStorage{Err: err}) } // Using MatrixToSeriesSet (and in turn NewConcreteSeriesSet), sorts the series. - return series.MatrixToSeriesSet(matrix), nil, nil + return series.MatrixToSeriesSet(matrix) } -func (q *distributorQuerier) streamingSelect(sp storage.SelectHints, matchers []*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (q *distributorQuerier) streamingSelect(sp storage.SelectHints, matchers []*labels.Matcher) storage.SeriesSet { userID, err := user.ExtractOrgID(q.ctx) if err != nil { - return nil, nil, promql.ErrStorage{Err: err} + return storage.ErrSeriesSet(promql.ErrStorage{Err: err}) } mint, maxt := sp.Start, sp.End results, err := q.distributor.QueryStream(q.ctx, model.Time(mint), model.Time(maxt), matchers...) if err != nil { - return nil, nil, promql.ErrStorage{Err: err} + return storage.ErrSeriesSet(promql.ErrStorage{Err: err}) } if len(results.Timeseries) != 0 { - return newTimeSeriesSeriesSet(results.Timeseries), nil, nil + return newTimeSeriesSeriesSet(results.Timeseries) } serieses := make([]storage.Series, 0, len(results.Chunkseries)) @@ -110,7 +131,7 @@ func (q *distributorQuerier) streamingSelect(sp storage.SelectHints, matchers [] chunks, err := chunkcompat.FromChunks(userID, ls, result.Chunks) if err != nil { - return nil, nil, promql.ErrStorage{Err: err} + return storage.ErrSeriesSet(promql.ErrStorage{Err: err}) } series := &chunkSeries{ @@ -121,7 +142,7 @@ func (q *distributorQuerier) streamingSelect(sp storage.SelectHints, matchers [] serieses = append(serieses, series) } - return series.NewConcreteSeriesSet(serieses), nil, nil + return series.NewConcreteSeriesSet(serieses) } func (q *distributorQuerier) LabelValues(name string) ([]string, storage.Warnings, error) { diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/frontend/frontend.go b/vendor/github.com/cortexproject/cortex/pkg/querier/frontend/frontend.go index f2fba3c8191a5..5d1e5fc81bbcc 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/frontend/frontend.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/frontend/frontend.go @@ -189,7 +189,7 @@ func (f *Frontend) handle(w http.ResponseWriter, r *http.Request) { // Ensure the form has been parsed so all the parameters are present err = r.ParseForm() if err != nil { - level.Warn(util.WithContext(r.Context(), f.log)).Log("msg", "unable to parse form for request", "error", err) + level.Warn(util.WithContext(r.Context(), f.log)).Log("msg", "unable to parse form for request", "err", err) } // Attempt to iterate through the Form to log any filled in values diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/lazyquery/lazyquery.go b/vendor/github.com/cortexproject/cortex/pkg/querier/lazyquery/lazyquery.go index e0e03f130d84f..82d6bc6f7924c 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/lazyquery/lazyquery.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/lazyquery/lazyquery.go @@ -43,27 +43,18 @@ func NewLazyQuerier(next storage.Querier) storage.Querier { return LazyQuerier{next} } -func (l LazyQuerier) createSeriesSet(selectSorted bool, params *storage.SelectHints, matchers []*labels.Matcher) chan storage.SeriesSet { +// Select implements Storage.Querier +func (l LazyQuerier) Select(selectSorted bool, params *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { // make sure there is space in the buffer, to unblock the goroutine and let it die even if nobody is // waiting for the result yet (or anymore). future := make(chan storage.SeriesSet, 1) go func() { - set, _, err := l.next.Select(selectSorted, params, matchers...) - if err != nil { - future <- errSeriesSet{err} - } else { - future <- set - } + future <- l.next.Select(selectSorted, params, matchers...) }() - return future -} -// Select implements Storage.Querier -func (l LazyQuerier) Select(selectSorted bool, params *storage.SelectHints, matchers ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { - future := l.createSeriesSet(selectSorted, params, matchers) return &lazySeriesSet{ future: future, - }, nil, nil + } } // LabelValues implements Storage.Querier @@ -91,27 +82,6 @@ func (l LazyQuerier) Get(ctx context.Context, userID string, from, through model return store.Get(ctx, userID, from, through, matchers...) } -func NewErrSeriesSet(err error) storage.SeriesSet { - return errSeriesSet{err} -} - -// errSeriesSet implements storage.SeriesSet, just returning an error. -type errSeriesSet struct { - err error -} - -func (errSeriesSet) Next() bool { - return false -} - -func (errSeriesSet) At() storage.Series { - return nil -} - -func (e errSeriesSet) Err() error { - return e.err -} - type lazySeriesSet struct { next storage.SeriesSet future chan storage.SeriesSet @@ -125,6 +95,7 @@ func (s *lazySeriesSet) Next() bool { return s.next.Next() } +// At implements storage.SeriesSet. func (s lazySeriesSet) At() storage.Series { if s.next == nil { s.next = <-s.future @@ -132,9 +103,15 @@ func (s lazySeriesSet) At() storage.Series { return s.next.At() } +// Err implements storage.SeriesSet. func (s lazySeriesSet) Err() error { if s.next == nil { s.next = <-s.future } return s.next.Err() } + +// Warnings implements storage.SeriesSet. +func (s lazySeriesSet) Warnings() storage.Warnings { + return nil +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/querier.go b/vendor/github.com/cortexproject/cortex/pkg/querier/querier.go index 66fd3e5be652a..95681617fbd69 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/querier.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/querier.go @@ -59,9 +59,11 @@ type Config struct { legacyLookbackDelta time.Duration // Blocks storage only. - StoreGatewayAddresses string `yaml:"store_gateway_addresses"` - StoreGatewayClient tls.ClientConfig `yaml:"store_gateway_client"` - BlocksConsistencyCheck BlocksConsistencyCheckConfig `yaml:"blocks_consistency_check" doc:"description=Configures the consistency check done by the querier on queried blocks when running the experimental blocks storage."` + StoreGatewayAddresses string `yaml:"store_gateway_addresses"` + StoreGatewayClient tls.ClientConfig `yaml:"store_gateway_client"` + + SecondStoreEngine string `yaml:"second_store_engine"` + UseSecondStoreBeforeTime flagext.Time `yaml:"use_second_store_before_time"` } var ( @@ -75,7 +77,6 @@ const ( // RegisterFlags adds the flags required to config this to the given FlagSet. func (cfg *Config) RegisterFlags(f *flag.FlagSet) { cfg.StoreGatewayClient.RegisterFlagsWithPrefix("experimental.querier.store-gateway-client", f) - cfg.BlocksConsistencyCheck.RegisterFlagsWithPrefix("experimental.querier.blocks-consistency-check", f) f.IntVar(&cfg.MaxConcurrent, "querier.max-concurrent", 20, "The maximum number of concurrent queries.") f.DurationVar(&cfg.Timeout, "querier.timeout", 2*time.Minute, "The timeout for a query.") f.BoolVar(&cfg.Iterators, "querier.iterators", false, "Use iterators to execute query, as opposed to fully materialising the series in memory.") @@ -91,6 +92,8 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&cfg.LookbackDelta, "querier.lookback-delta", defaultLookbackDelta, "Time since the last sample after which a time series is considered stale and ignored by expression evaluations.") // TODO: Remove this flag in v1.4.0. f.DurationVar(&cfg.legacyLookbackDelta, "promql.lookback-delta", defaultLookbackDelta, "[DEPRECATED] Time since the last sample after which a time series is considered stale and ignored by expression evaluations. Please use -querier.lookback-delta instead.") + f.StringVar(&cfg.SecondStoreEngine, "querier.second-store-engine", "", "Second store engine to use for querying. Empty = disabled.") + f.Var(&cfg.UseSecondStoreBeforeTime, "querier.use-second-store-before-time", "If specified, second store is only used for queries before this timestamp. Default value 0 means secondary store is always queried.") } // Validate the config @@ -129,12 +132,20 @@ func NewChunkStoreQueryable(cfg Config, chunkStore chunkstore.ChunkStore) storag } // New builds a queryable and promql engine. -func New(cfg Config, distributor Distributor, storeQueryable storage.Queryable, tombstonesLoader *purger.TombstonesLoader, reg prometheus.Registerer) (storage.Queryable, *promql.Engine) { +func New(cfg Config, distributor Distributor, stores []QueryableWithFilter, tombstonesLoader *purger.TombstonesLoader, reg prometheus.Registerer) (storage.Queryable, *promql.Engine) { iteratorFunc := getChunksIteratorFunction(cfg) - var queryable storage.Queryable - distributorQueryable := newDistributorQueryable(distributor, cfg.IngesterStreaming, iteratorFunc) - queryable = NewQueryable(distributorQueryable, storeQueryable, iteratorFunc, cfg, tombstonesLoader) + distributorQueryable := newDistributorQueryable(distributor, cfg.IngesterStreaming, iteratorFunc, cfg.QueryIngestersWithin) + + ns := make([]QueryableWithFilter, len(stores)) + for ix, s := range stores { + ns[ix] = storeQueryable{ + QueryableWithFilter: s, + QueryStoreAfter: cfg.QueryStoreAfter, + } + } + + queryable := NewQueryable(distributorQueryable, ns, iteratorFunc, cfg, tombstonesLoader) lazyQueryable := storage.QueryableFunc(func(ctx context.Context, mint int64, maxt int64) (storage.Querier, error) { querier, err := queryable.Querier(ctx, mint, maxt) @@ -176,8 +187,17 @@ func createActiveQueryTracker(cfg Config) *promql.ActiveQueryTracker { return nil } +// QueryableWithFilter extends Queryable interface with `UseQueryable` filtering function. +type QueryableWithFilter interface { + storage.Queryable + + // UseQueryable returns true if this queryable should be used to satisfy the query for given time range. + // Query min and max time are in milliseconds since epoch. + UseQueryable(now time.Time, queryMinT, queryMaxT int64) bool +} + // NewQueryable creates a new Queryable for cortex. -func NewQueryable(distributor, store storage.Queryable, chunkIterFn chunkIteratorFunc, cfg Config, tombstonesLoader *purger.TombstonesLoader) storage.Queryable { +func NewQueryable(distributor QueryableWithFilter, stores []QueryableWithFilter, chunkIterFn chunkIteratorFunc, cfg Config, tombstonesLoader *purger.TombstonesLoader) storage.Queryable { return storage.QueryableFunc(func(ctx context.Context, mint, maxt int64) (storage.Querier, error) { now := time.Now() @@ -207,14 +227,16 @@ func NewQueryable(distributor, store storage.Queryable, chunkIterFn chunkIterato q.metadataQuerier = dqr - // Include ingester only if maxt is within QueryIngestersWithin w.r.t. current time. - if cfg.QueryIngestersWithin == 0 || maxt >= util.TimeToMillis(now.Add(-cfg.QueryIngestersWithin)) { + if distributor.UseQueryable(now, mint, maxt) { q.queriers = append(q.queriers, dqr) } - // Include store only if mint is within QueryStoreAfter w.r.t current time. - if cfg.QueryStoreAfter == 0 || mint <= util.TimeToMillis(now.Add(-cfg.QueryStoreAfter)) { - cqr, err := store.Querier(ctx, mint, maxt) + for _, s := range stores { + if !s.UseQueryable(now, mint, maxt) { + continue + } + + cqr, err := s.Querier(ctx, mint, maxt) if err != nil { return nil, err } @@ -242,7 +264,7 @@ type querier struct { // Select implements storage.Querier interface. // The bool passed is ignored because the series is always sorted. -func (q querier) Select(_ bool, sp *storage.SelectHints, matchers ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (q querier) Select(_ bool, sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { log, ctx := spanlogger.New(q.ctx, "querier.Select") defer log.Span.Finish() @@ -260,49 +282,38 @@ func (q querier) Select(_ bool, sp *storage.SelectHints, matchers ...*labels.Mat userID, err := user.ExtractOrgID(ctx) if err != nil { - return nil, nil, promql.ErrStorage{Err: err} + return storage.ErrSeriesSet(promql.ErrStorage{Err: err}) } tombstones, err := q.tombstonesLoader.GetPendingTombstonesForInterval(userID, model.Time(sp.Start), model.Time(sp.End)) if err != nil { - return nil, nil, promql.ErrStorage{Err: err} + return storage.ErrSeriesSet(promql.ErrStorage{Err: err}) } if len(q.queriers) == 1 { - seriesSet, warning, err := q.queriers[0].Select(true, sp, matchers...) - if err != nil { - return nil, warning, err - } + seriesSet := q.queriers[0].Select(true, sp, matchers...) if tombstones.Len() != 0 { seriesSet = series.NewDeletedSeriesSet(seriesSet, tombstones, model.Interval{Start: model.Time(sp.Start), End: model.Time(sp.End)}) } - return seriesSet, warning, nil + return seriesSet } sets := make(chan storage.SeriesSet, len(q.queriers)) - errs := make(chan error, len(q.queriers)) for _, querier := range q.queriers { go func(querier storage.Querier) { - set, _, err := querier.Select(true, sp, matchers...) - if err != nil { - errs <- err - } else { - sets <- set - } + sets <- querier.Select(true, sp, matchers...) }(querier) } var result []storage.SeriesSet for range q.queriers { select { - case err := <-errs: - return nil, nil, err case set := <-sets: result = append(result, set) case <-ctx.Done(): - return nil, nil, ctx.Err() + return storage.ErrSeriesSet(ctx.Err()) } } @@ -314,7 +325,7 @@ func (q querier) Select(_ bool, sp *storage.SelectHints, matchers ...*labels.Mat if tombstones.Len() != 0 { seriesSet = series.NewDeletedSeriesSet(seriesSet, tombstones, model.Interval{Start: model.Time(sp.Start), End: model.Time(sp.End)}) } - return seriesSet, nil, nil + return seriesSet } // LabelsValue implements storage.Querier. @@ -340,10 +351,10 @@ func (q querier) mergeSeriesSets(sets []storage.SeriesSet) storage.SeriesSet { for _, set := range sets { if !set.Next() { // nothing in this set. If it has no error, we can ignore it completely. - // If there is error, we better report it. + // If there is error, we have to report it. err := set.Err() if err != nil { - otherSets = append(otherSets, lazyquery.NewErrSeriesSet(err)) + otherSets = append(otherSets, storage.ErrSeriesSet(err)) } continue } @@ -411,3 +422,61 @@ func (pss *seriesSetWithFirstSeries) Err() error { } return pss.set.Err() } + +func (pss *seriesSetWithFirstSeries) Warnings() storage.Warnings { + if pss.firstSeries != nil { + return nil + } + return pss.set.Warnings() +} + +type storeQueryable struct { + QueryableWithFilter + QueryStoreAfter time.Duration +} + +func (s storeQueryable) UseQueryable(now time.Time, queryMinT, queryMaxT int64) bool { + // Include this store only if mint is within QueryStoreAfter w.r.t current time. + if s.QueryStoreAfter != 0 && queryMinT > util.TimeToMillis(now.Add(-s.QueryStoreAfter)) { + return false + } + return s.QueryableWithFilter.UseQueryable(now, queryMinT, queryMaxT) +} + +type alwaysTrueFilterQueryable struct { + storage.Queryable +} + +func (alwaysTrueFilterQueryable) UseQueryable(_ time.Time, _, _ int64) bool { + return true +} + +// Wraps storage.Queryable into QueryableWithFilter, with no query filtering. +func UseAlwaysQueryable(q storage.Queryable) QueryableWithFilter { + return alwaysTrueFilterQueryable{Queryable: q} +} + +type useBeforeTimestampQueryable struct { + storage.Queryable + ts int64 // Timestamp in milliseconds +} + +func (u useBeforeTimestampQueryable) UseQueryable(_ time.Time, queryMinT, _ int64) bool { + if u.ts == 0 { + return true + } + return queryMinT < u.ts +} + +// Returns QueryableWithFilter, that is used only if query starts before given timestamp. +// If timestamp is zero (time.IsZero), queryable is always used. +func UseBeforeTimestampQueryable(queryable storage.Queryable, ts time.Time) QueryableWithFilter { + t := int64(0) + if !ts.IsZero() { + t = util.TimeToMillis(ts) + } + return useBeforeTimestampQueryable{ + Queryable: queryable, + ts: t, + } +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/queryable.go b/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/queryable.go index 19a7158d0115b..9b0a0b6e4d647 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/queryable.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/queryable.go @@ -30,8 +30,11 @@ func (q *ShardedQueryable) Querier(ctx context.Context, mint, maxt int64) (stora return q.sharededQuerier, nil } -func (q *ShardedQueryable) getResponseHeaders() map[string][]string { - return q.sharededQuerier.ResponseHeaders +func (q *ShardedQueryable) getResponseHeaders() []*PrometheusResponseHeader { + q.sharededQuerier.ResponseHeadersMtx.Lock() + defer q.sharededQuerier.ResponseHeadersMtx.Unlock() + + return headersMapToPrometheusResponseHeaders(q.sharededQuerier.ResponseHeaders) } // ShardedQuerier is a an implementor of the Querier interface. @@ -45,7 +48,7 @@ type ShardedQuerier struct { // Select returns a set of series that matches the given label matchers. // The bool passed is ignored because the series is always sorted. -func (q *ShardedQuerier) Select(_ bool, _ *storage.SelectHints, matchers ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (q *ShardedQuerier) Select(_ bool, _ *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { var embeddedQuery string var isEmbedded bool for _, matcher := range matchers { @@ -62,18 +65,18 @@ func (q *ShardedQuerier) Select(_ bool, _ *storage.SelectHints, matchers ...*lab if embeddedQuery != "" { return q.handleEmbeddedQuery(embeddedQuery) } - return nil, nil, errors.Errorf(missingEmbeddedQueryMsg) + return storage.ErrSeriesSet(errors.Errorf(missingEmbeddedQueryMsg)) } - return nil, nil, errors.Errorf(nonEmbeddedErrMsg) + return storage.ErrSeriesSet(errors.Errorf(nonEmbeddedErrMsg)) } // handleEmbeddedQuery defers execution of an encoded query to a downstream Handler -func (q *ShardedQuerier) handleEmbeddedQuery(encoded string) (storage.SeriesSet, storage.Warnings, error) { +func (q *ShardedQuerier) handleEmbeddedQuery(encoded string) storage.SeriesSet { queries, err := astmapper.JSONCodec.Decode(encoded) if err != nil { - return nil, nil, err + return storage.ErrSeriesSet(err) } ctx, cancel := context.WithCancel(q.Ctx) @@ -95,8 +98,8 @@ func (q *ShardedQuerier) handleEmbeddedQuery(encoded string) (storage.SeriesSet, errCh <- err return } - samplesCh <- streams q.setResponseHeaders(resp.(*PrometheusResponse).Headers) + samplesCh <- streams }(query) } @@ -105,13 +108,13 @@ func (q *ShardedQuerier) handleEmbeddedQuery(encoded string) (storage.SeriesSet, for i := 0; i < len(queries); i++ { select { case err := <-errCh: - return nil, nil, err + return storage.ErrSeriesSet(err) case streams := <-samplesCh: samples = append(samples, streams...) } } - return NewSeriesSet(samples), nil, err + return NewSeriesSet(samples) } func (q *ShardedQuerier) setResponseHeaders(headers []*PrometheusResponseHeader) { @@ -141,3 +144,11 @@ func (q *ShardedQuerier) LabelNames() ([]string, storage.Warnings, error) { func (q *ShardedQuerier) Close() error { return nil } + +func headersMapToPrometheusResponseHeaders(headersMap map[string][]string) (prs []*PrometheusResponseHeader) { + for h, v := range headersMap { + prs = append(prs, &PrometheusResponseHeader{Name: h, Values: v}) + } + + return +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/querysharding.go b/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/querysharding.go index 1460890ec1e0e..046212eca8782 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/querysharding.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/querysharding.go @@ -236,7 +236,7 @@ func (qs *queryShard) Do(ctx context.Context, r Request) (Response, error) { ResultType: string(res.Value.Type()), Result: extracted, }, - Headers: headersMapToPrometheusResponseHeaders(shardedQueryable.getResponseHeaders()), + Headers: shardedQueryable.getResponseHeaders(), }, nil } @@ -322,11 +322,3 @@ func partitionRequest(r Request, t time.Time) (before Request, after Request) { return r.WithStartEnd(r.GetStart(), boundary), r.WithStartEnd(boundary, r.GetEnd()) } - -func headersMapToPrometheusResponseHeaders(headersMap map[string][]string) (prs []*PrometheusResponseHeader) { - for h, v := range headersMap { - prs = append(prs, &PrometheusResponseHeader{Name: h, Values: v}) - } - - return -} diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/test_utils.go b/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/test_utils.go index a76fb417afdf6..dae9eee0db647 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/test_utils.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/test_utils.go @@ -84,12 +84,12 @@ func (q *MockShardedQueryable) Querier(ctx context.Context, mint, maxt int64) (s // Select implements storage.Querier interface. // The bool passed is ignored because the series is always sorted. -func (q *MockShardedQueryable) Select(_ bool, _ *storage.SelectHints, matchers ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (q *MockShardedQueryable) Select(_ bool, _ *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { tStart := time.Now() shard, _, err := astmapper.ShardFromMatchers(matchers) if err != nil { - return nil, nil, err + return storage.ErrSeriesSet(err) } var ( @@ -141,7 +141,7 @@ func (q *MockShardedQueryable) Select(_ bool, _ *storage.SelectHints, matchers . } // sorted - return series.NewConcreteSeriesSet(results), nil, nil + return series.NewConcreteSeriesSet(results) } // ShardLabelSeries allows extending a Series with new labels. This is helpful for adding cortex shard labels diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/remote_read.go b/vendor/github.com/cortexproject/cortex/pkg/querier/remote_read.go index 036a4ce6ff9df..cf935007cd3b6 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/remote_read.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/remote_read.go @@ -50,12 +50,7 @@ func RemoteReadHandler(q storage.Queryable) http.Handler { Start: int64(from), End: int64(to), } - seriesSet, _, err := querier.Select(false, params, matchers...) - if err != nil { - errors <- err - return - } - + seriesSet := querier.Select(false, params, matchers...) resp.Results[i], err = seriesSetToQueryResponse(seriesSet) errors <- err }(i, qr) diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/series/series_set.go b/vendor/github.com/cortexproject/cortex/pkg/querier/series/series_set.go index 4dca6163b6612..0fdfb109bbc56 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/series/series_set.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/series/series_set.go @@ -44,22 +44,27 @@ func NewConcreteSeriesSet(series []storage.Series) storage.SeriesSet { } } -// Next iterates through a series set and impls storage.SeriesSet +// Next iterates through a series set and implements storage.SeriesSet. func (c *ConcreteSeriesSet) Next() bool { c.cur++ return c.cur < len(c.series) } -// At returns the current series and impls storage.SeriesSet +// At returns the current series and implements storage.SeriesSet. func (c *ConcreteSeriesSet) At() storage.Series { return c.series[c.cur] } -// Err impls storage.SeriesSet +// Err implements storage.SeriesSet. func (c *ConcreteSeriesSet) Err() error { return nil } +// Warnings implements storage.SeriesSet. +func (c *ConcreteSeriesSet) Warnings() storage.Warnings { + return nil +} + // ConcreteSeries implements storage.Series. type ConcreteSeries struct { labels labels.Labels @@ -74,12 +79,12 @@ func NewConcreteSeries(ls labels.Labels, samples []model.SamplePair) *ConcreteSe } } -// Labels impls storage.Series +// Labels implements storage.Series func (c *ConcreteSeries) Labels() labels.Labels { return c.labels } -// Iterator impls storage.Series +// Iterator implements storage.Series func (c *ConcreteSeries) Iterator() chunkenc.Iterator { return NewConcreteSeriesIterator(c) } @@ -224,6 +229,10 @@ func (d DeletedSeriesSet) Err() error { return d.seriesSet.Err() } +func (d DeletedSeriesSet) Warnings() storage.Warnings { + return nil +} + type DeletedSeries struct { series storage.Series deletedIntervals []model.Interval @@ -347,13 +356,30 @@ func (emptySeriesIterator) Err() error { return nil } -type emptySeriesSet struct{} +type seriesSetWithWarnings struct { + wrapped storage.SeriesSet + warnings storage.Warnings +} -func (emptySeriesSet) Next() bool { return false } -func (emptySeriesSet) At() storage.Series { return nil } -func (emptySeriesSet) Err() error { return nil } +func NewSeriesSetWithWarnings(wrapped storage.SeriesSet, warnings storage.Warnings) storage.SeriesSet { + return seriesSetWithWarnings{ + wrapped: wrapped, + warnings: warnings, + } +} + +func (s seriesSetWithWarnings) Next() bool { + return s.wrapped.Next() +} + +func (s seriesSetWithWarnings) At() storage.Series { + return s.wrapped.At() +} + +func (s seriesSetWithWarnings) Err() error { + return s.wrapped.Err() +} -// NewEmptySeriesSet returns a new series set that contains no series. -func NewEmptySeriesSet() storage.SeriesSet { - return emptySeriesSet{} +func (s seriesSetWithWarnings) Warnings() storage.Warnings { + return append(s.wrapped.Warnings(), s.warnings...) } diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/timeseries_series_set.go b/vendor/github.com/cortexproject/cortex/pkg/querier/timeseries_series_set.go index 53d5d3aba53fe..7d4df711fc823 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/timeseries_series_set.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/timeseries_series_set.go @@ -24,10 +24,10 @@ func newTimeSeriesSeriesSet(series []client.TimeSeries) *timeSeriesSeriesSet { } } -// Next implements SeriesSet interface +// Next implements storage.SeriesSet interface. func (t *timeSeriesSeriesSet) Next() bool { t.i++; return t.i < len(t.ts) } -// At implements SeriesSet interface +// At implements storage.SeriesSet interface. func (t *timeSeriesSeriesSet) At() storage.Series { if t.i < 0 { return nil @@ -35,9 +35,12 @@ func (t *timeSeriesSeriesSet) At() storage.Series { return ×eries{series: t.ts[t.i]} } -// Err implements SeriesSet interface +// Err implements storage.SeriesSet interface. func (t *timeSeriesSeriesSet) Err() error { return nil } +// Warnings implements storage.SeriesSet interface. +func (t *timeSeriesSeriesSet) Warnings() storage.Warnings { return nil } + // timeseries is a type wrapper that implements the storage.Series interface type timeseries struct { series client.TimeSeries diff --git a/vendor/github.com/cortexproject/cortex/pkg/ruler/ruler.go b/vendor/github.com/cortexproject/cortex/pkg/ruler/ruler.go index faec4731a2d41..6bc0fb3ee60e6 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/ruler/ruler.go +++ b/vendor/github.com/cortexproject/cortex/pkg/ruler/ruler.go @@ -88,6 +88,13 @@ type Config struct { // HTTP timeout duration when sending notifications to the Alertmanager. NotificationTimeout time.Duration `yaml:"notification_timeout"` + // Max time to tolerate outage for restoring "for" state of alert. + OutageTolerance time.Duration `yaml:"for_outage_tolerance"` + // Minimum duration between alert and restored "for" state. This is maintained only for alerts with configured "for" time greater than grace period. + ForGracePeriod time.Duration `yaml:"for_grace_period"` + // Minimum amount of time to wait before resending an alert to Alertmanager. + ResendDelay time.Duration `yaml:"resend_delay"` + // Enable sharding rule groups. EnableSharding bool `yaml:"enable_sharding"` SearchPendingFor time.Duration `yaml:"search_pending_for"` @@ -132,6 +139,9 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&cfg.FlushCheckPeriod, "ruler.flush-period", 1*time.Minute, "Period with which to attempt to flush rule groups.") f.StringVar(&cfg.RulePath, "ruler.rule-path", "/rules", "file path to store temporary rule files for the prometheus rule managers") f.BoolVar(&cfg.EnableAPI, "experimental.ruler.enable-api", false, "Enable the ruler api") + f.DurationVar(&cfg.OutageTolerance, "ruler.for-outage-tolerance", time.Hour, `Max time to tolerate outage for restoring "for" state of alert.`) + f.DurationVar(&cfg.ForGracePeriod, "ruler.for-grace-period", 10*time.Minute, `Minimum duration between alert and restored "for" state. This is maintained only for alerts with configured "for" time greater than grace period.`) + f.DurationVar(&cfg.ResendDelay, "ruler.resend-delay", time.Minute, `Minimum amount of time to wait before resending an alert to Alertmanager.`) } // Ruler evaluates rules. @@ -523,14 +533,17 @@ func (r *Ruler) newManager(ctx context.Context, userID string) (*promRules.Manag reg = prometheus.WrapRegistererWithPrefix("cortex_", reg) logger := log.With(r.logger, "user", userID) opts := &promRules.ManagerOptions{ - Appendable: tsdb, - TSDB: tsdb, - QueryFunc: engineQueryFunc(r.engine, r.queryable, r.cfg.EvaluationDelay), - Context: user.InjectOrgID(ctx, userID), - ExternalURL: r.alertURL, - NotifyFunc: sendAlerts(notifier, r.alertURL.String()), - Logger: logger, - Registerer: reg, + Appendable: tsdb, + TSDB: tsdb, + QueryFunc: engineQueryFunc(r.engine, r.queryable, r.cfg.EvaluationDelay), + Context: user.InjectOrgID(ctx, userID), + ExternalURL: r.alertURL, + NotifyFunc: sendAlerts(notifier, r.alertURL.String()), + Logger: logger, + Registerer: reg, + OutageTolerance: r.cfg.OutageTolerance, + ForGracePeriod: r.cfg.ForGracePeriod, + ResendDelay: r.cfg.ResendDelay, } return promRules.NewManager(opts), nil } diff --git a/vendor/github.com/cortexproject/cortex/pkg/util/flagext/time.go b/vendor/github.com/cortexproject/cortex/pkg/util/flagext/time.go new file mode 100644 index 0000000000000..452857e9de8f1 --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/util/flagext/time.go @@ -0,0 +1,60 @@ +package flagext + +import ( + "fmt" + "time" +) + +// Time usable as flag or in YAML config. +type Time time.Time + +// String implements flag.Value +func (t Time) String() string { + if time.Time(t).IsZero() { + return "0" + } + + return time.Time(t).Format(time.RFC3339) +} + +// Set implements flag.Value +func (t *Time) Set(s string) error { + if s == "0" { + *t = Time(time.Time{}) + return nil + } + + p, err := time.Parse("2006-01-02", s) + if err == nil { + *t = Time(p) + return nil + } + + p, err = time.Parse("2006-01-02T15:04", s) + if err == nil { + *t = Time(p) + return nil + } + + p, err = time.Parse("2006-01-02T15:04:05Z07:00", s) + if err == nil { + *t = Time(p) + return nil + } + + return fmt.Errorf("failed to parse time: %q", s) +} + +// UnmarshalYAML implements yaml.Unmarshaler. +func (t *Time) UnmarshalYAML(unmarshal func(interface{}) error) error { + var s string + if err := unmarshal(&s); err != nil { + return err + } + return t.Set(s) +} + +// MarshalYAML implements yaml.Marshaler. +func (t Time) MarshalYAML() (interface{}, error) { + return t.String(), nil +} diff --git a/vendor/github.com/digitalocean/godo/.gitignore b/vendor/github.com/digitalocean/godo/.gitignore new file mode 100644 index 0000000000000..48b8bf9072d87 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/.gitignore @@ -0,0 +1 @@ +vendor/ diff --git a/vendor/github.com/digitalocean/godo/.whitesource b/vendor/github.com/digitalocean/godo/.whitesource new file mode 100644 index 0000000000000..e0aaa3e9ebaae --- /dev/null +++ b/vendor/github.com/digitalocean/godo/.whitesource @@ -0,0 +1,8 @@ +{ + "checkRunSettings": { + "vulnerableCheckRunConclusionLevel": "failure" + }, + "issueSettings": { + "minSeverityLevel": "LOW" + } +} \ No newline at end of file diff --git a/vendor/github.com/digitalocean/godo/1-click.go b/vendor/github.com/digitalocean/godo/1-click.go new file mode 100644 index 0000000000000..fab04fe510f4f --- /dev/null +++ b/vendor/github.com/digitalocean/godo/1-click.go @@ -0,0 +1,52 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +const oneClickBasePath = "v2/1-clicks" + +// OneClickService is an interface for interacting with 1-clicks with the +// DigitalOcean API. +// See: https://developers.digitalocean.com/documentation/v2/#1-click-applications +type OneClickService interface { + List(context.Context, string) ([]*OneClick, *Response, error) +} + +var _ OneClickService = &OneClickServiceOp{} + +// OneClickServiceOp interfaces with 1-click endpoints in the DigitalOcean API. +type OneClickServiceOp struct { + client *Client +} + +// OneClick is the structure of a 1-click +type OneClick struct { + Slug string `json:"slug"` + Type string `json:"type"` +} + +// OneClicksRoot is the root of the json payload that contains a list of 1-clicks +type OneClicksRoot struct { + List []*OneClick `json:"1_clicks"` +} + +// List returns a list of the available 1-click applications. +func (ocs *OneClickServiceOp) List(ctx context.Context, oneClickType string) ([]*OneClick, *Response, error) { + path := fmt.Sprintf(`%s?type=%s`, oneClickBasePath, oneClickType) + + req, err := ocs.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(OneClicksRoot) + resp, err := ocs.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.List, resp, nil +} diff --git a/vendor/github.com/digitalocean/godo/CHANGELOG.md b/vendor/github.com/digitalocean/godo/CHANGELOG.md new file mode 100644 index 0000000000000..c826205fac109 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/CHANGELOG.md @@ -0,0 +1,255 @@ +# Change Log + +## [v1.37.0] - 2020-06-01 + +- #336 registry: URL encode repository names when building URLs. @adamwg +- #335 Add 1-click service and request. @scottcrawford03 + +## [v1.36.0] - 2020-05-12 + +- #331 Expose expiry_seconds for Registry.DockerCredentials. @andrewsomething + +## [v1.35.1] - 2020-04-21 + +- #328 Update vulnerable x/crypto dependency - @bentranter + +## [v1.35.0] - 2020-04-20 + +- #326 Add TagCount field to registry/Repository - @nicktate +- #325 Add DOCR EA routes - @nicktate +- #324 Upgrade godo to Go 1.14 - @bentranter + +## [v1.34.0] - 2020-03-30 + +- #320 Add VPC v3 attributes - @viola + +## [v1.33.1] - 2020-03-23 + +- #318 upgrade github.com/stretchr/objx past 0.1.1 - @hilary + +## [v1.33.0] - 2020-03-20 + +- #310 Add BillingHistory service and List endpoint - @rbutler +- #316 load balancers: add new enable_backend_keepalive field - @anitgandhi + +## [v1.32.0] - 2020-03-04 + +- #311 Add reset database user auth method - @zbarahal-do + +## [v1.31.0] - 2020-02-28 + +- #305 invoices: GetPDF and GetCSV methods - @rbutler +- #304 Add NewFromToken convenience method to init client - @bentranter +- #301 invoices: Get, Summary, and List methods - @rbutler +- #299 Fix param expiry_seconds for kubernetes.GetCredentials request - @velp + +## [v1.30.0] - 2020-02-03 + +- #295 registry: support the created_at field - @adamwg +- #293 doks: node pool labels - @snormore + +## [v1.29.0] - 2019-12-13 + +- #288 Add Balance Get method - @rbutler +- #286,#289 Deserialize meta field - @timoreimann + +## [v1.28.0] - 2019-12-04 + +- #282 Add valid Redis eviction policy constants - @bentranter +- #281 Remove databases info from top-level godoc string - @bentranter +- #280 Fix VolumeSnapshotResourceType value volumesnapshot -> volume_snapshot - @aqche + +## [v1.27.0] - 2019-11-18 + +- #278 add mysql user auth settings for database users - @gregmankes + +## [v1.26.0] - 2019-11-13 + +- #272 dbaas: get and set mysql sql mode - @mikejholly + +## [v1.25.0] - 2019-11-13 + +- #275 registry/docker-credentials: add support for the read/write parameter - @kamaln7 +- #273 implement the registry/docker-credentials endpoint - @kamaln7 +- #271 Add registry resource - @snormore + +## [v1.24.1] - 2019-11-04 + +- #264 Update isLast to check p.Next - @aqche + +## [v1.24.0] - 2019-10-30 + +- #267 Return []DatabaseFirewallRule in addition to raw response. - @andrewsomething + +## [v1.23.1] - 2019-10-30 + +- #265 add support for getting/setting firewall rules - @gregmankes +- #262 remove ResolveReference call - @mdanzinger +- #261 Update CONTRIBUTING.md - @mdanzinger + +## [v1.22.0] - 2019-09-24 + +- #259 Add Kubernetes GetCredentials method - @snormore + +## [v1.21.1] - 2019-09-19 + +- #257 Upgrade to Go 1.13 - @bentranter + +## [v1.21.0] - 2019-09-16 + +- #255 Add DropletID to Kubernetes Node instance - @snormore +- #254 Add tags to Database, DatabaseReplica - @Zyqsempai + +## [v1.20.0] - 2019-09-06 + +- #252 Add Kubernetes autoscale config fields - @snormore +- #251 Support unset fields on Kubernetes cluster and node pool updates - @snormore +- #250 Add Kubernetes GetUser method - @snormore + +## [v1.19.0] - 2019-07-19 + +- #244 dbaas: add private-network-uuid field to create request + +## [v1.18.0] - 2019-07-17 + +- #241 Databases: support for custom VPC UUID on migrate @mikejholly +- #240 Add the ability to get URN for a Database @stack72 +- #236 Fix omitempty typos in JSON struct tags @amccarthy1 + +## [v1.17.0] - 2019-06-21 + +- #238 Add support for Redis eviction policy in Databases @mikejholly + +## [v1.16.0] - 2019-06-04 + +- #233 Add Kubernetes DeleteNode method, deprecate RecycleNodePoolNodes @bouk + +## [v1.15.0] - 2019-05-13 + +- #231 Add private connection fields to Databases - @mikejholly +- #223 Introduce Go modules - @andreiavrammsd + +## [v1.14.0] - 2019-05-13 + +- #229 Add support for upgrading Kubernetes clusters - @adamwg + +## [v1.13.0] - 2019-04-19 + +- #213 Add tagging support for volume snapshots - @jcodybaker + +## [v1.12.0] - 2019-04-18 + +- #224 Add maintenance window support for Kubernetes- @fatih + +## [v1.11.1] - 2019-04-04 + +- #222 Fix Create Database Pools json fields - @sunny-b + +## [v1.11.0] - 2019-04-03 + +- #220 roll out vpc functionality - @jheimann + +## [v1.10.1] - 2019-03-27 + +- #219 Fix Database Pools json field - @sunny-b + +## [v1.10.0] - 2019-03-20 + +- #215 Add support for Databases - @mikejholly + +## [v1.9.0] - 2019-03-18 + +- #214 add support for enable_proxy_protocol. - @mregmi + +## [v1.8.0] - 2019-03-13 + +- #210 Expose tags on storage volume create/list/get. - @jcodybaker + +## [v1.7.5] - 2019-03-04 + +- #207 Add support for custom subdomains for Spaces CDN [beta] - @xornivore + +## [v1.7.4] - 2019-02-08 + +- #202 Allow tagging volumes - @mchitten + +## [v1.7.3] - 2018-12-18 + +- #196 Expose tag support for creating Load Balancers. + +## [v1.7.2] - 2018-12-04 + +- #192 Exposes more options for Kubernetes clusters. + +## [v1.7.1] - 2018-11-27 + +- #190 Expose constants for the state of Kubernetes clusters. + +## [v1.7.0] - 2018-11-13 + +- #188 Kubernetes support [beta] - @aybabtme + +## [v1.6.0] - 2018-10-16 + +- #185 Projects support [beta] - @mchitten + +## [v1.5.0] - 2018-10-01 + +- #181 Adding tagging images support - @hugocorbucci + +## [v1.4.2] - 2018-08-30 + +- #178 Allowing creating domain records with weight of 0 - @TFaga +- #177 Adding `VolumeLimit` to account - @lxfontes + +## [v1.4.1] - 2018-08-23 + +- #176 Fix cdn flush cache API endpoint - @sunny-b + +## [v1.4.0] - 2018-08-22 + +- #175 Add support for Spaces CDN - @sunny-b + +## [v1.3.0] - 2018-05-24 + +- #170 Add support for volume formatting - @adamwg + +## [v1.2.0] - 2018-05-08 + +- #166 Remove support for Go 1.6 - @iheanyi +- #165 Add support for Let's Encrypt Certificates - @viola + +## [v1.1.3] - 2018-03-07 + +- #156 Handle non-json errors from the API - @aknuds1 +- #158 Update droplet example to use latest instance type - @dan-v + +## [v1.1.2] - 2018-03-06 + +- #157 storage: list volumes should handle only name or only region params - @andrewsykim +- #154 docs: replace first example with fully-runnable example - @xmudrii +- #152 Handle flags & tag properties of domain record - @jaymecd + +## [v1.1.1] - 2017-09-29 + +- #151 Following user agent field recommendations - @joonas +- #148 AsRequest method to create load balancers requests - @lukegb + +## [v1.1.0] - 2017-06-06 + +### Added +- #145 Add FirewallsService for managing Firewalls with the DigitalOcean API. - @viola +- #139 Add TTL field to the Domains. - @xmudrii + +### Fixed +- #143 Fix oauth2.NoContext depreciation. - @jbowens +- #141 Fix DropletActions on tagged resources. - @xmudrii + +## [v1.0.0] - 2017-03-10 + +### Added +- #130 Add Convert to ImageActionsService. - @xmudrii +- #126 Add CertificatesService for managing certificates with the DigitalOcean API. - @viola +- #125 Add LoadBalancersService for managing load balancers with the DigitalOcean API. - @viola +- #122 Add GetVolumeByName to StorageService. - @protochron +- #113 Add context.Context to all calls. - @aybabtme diff --git a/vendor/github.com/digitalocean/godo/CONTRIBUTING.md b/vendor/github.com/digitalocean/godo/CONTRIBUTING.md new file mode 100644 index 0000000000000..33f03132a8187 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/CONTRIBUTING.md @@ -0,0 +1,54 @@ +# Contributing + +We love contributions! You are welcome to open a pull request, but it's a good idea to +open an issue and discuss your idea with us first. + +Once you are ready to open a PR, please keep the following guidelines in mind: + +1. Code should be `go fmt` compliant. +1. Types, structs and funcs should be documented. +1. Tests pass. + +## Getting set up + +`godo` uses go modules. Just fork this repo, clone your fork and off you go! + +## Running tests + +When working on code in this repository, tests can be run via: + +```sh +go test -mod=vendor . +``` + +## Versioning + +Godo follows [semver](https://www.semver.org) versioning semantics. +New functionality should be accompanied by increment to the minor +version number. Any code merged to master is subject to release. + +## Releasing + +Releasing a new version of godo is currently a manual process. + +Submit a separate pull request for the version change from the pull +request with your changes. + +1. Update the `CHANGELOG.md` with your changes. If a version header + for the next (unreleased) version does not exist, create one. + Include one bullet point for each piece of new functionality in the + release, including the pull request ID, description, and author(s). + +``` +## [v1.8.0] - 2019-03-13 + +- #210 Expose tags on storage volume create/list/get. - @jcodybaker +- #123 Update test dependencies - @digitalocean +``` + +2. Update the `libraryVersion` number in `godo.go`. +3. Make a pull request with these changes. This PR should be separate from the PR containing the godo changes. +4. Once the pull request has been merged, [draft a new release](https://github.com/digitalocean/godo/releases/new). +5. Update the `Tag version` and `Release title` field with the new godo version. Be sure the version has a `v` prefixed in both places. Ex `v1.8.0`. +6. Copy the changelog bullet points to the description field. +7. Publish the release. diff --git a/vendor/github.com/digitalocean/godo/LICENSE.txt b/vendor/github.com/digitalocean/godo/LICENSE.txt new file mode 100644 index 0000000000000..43c5d2eee7d96 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/LICENSE.txt @@ -0,0 +1,55 @@ +Copyright (c) 2014-2016 The godo AUTHORS. All rights reserved. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +====================== +Portions of the client are based on code at: +https://github.com/google/go-github/ + +Copyright (c) 2013 The go-github AUTHORS. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/vendor/github.com/digitalocean/godo/README.md b/vendor/github.com/digitalocean/godo/README.md new file mode 100644 index 0000000000000..cadeb69edbb9b --- /dev/null +++ b/vendor/github.com/digitalocean/godo/README.md @@ -0,0 +1,139 @@ +# Godo + +[![Build Status](https://travis-ci.org/digitalocean/godo.svg)](https://travis-ci.org/digitalocean/godo) +[![GoDoc](https://godoc.org/github.com/digitalocean/godo?status.svg)](https://godoc.org/github.com/digitalocean/godo) + +Godo is a Go client library for accessing the DigitalOcean V2 API. + +You can view the client API docs here: [http://godoc.org/github.com/digitalocean/godo](http://godoc.org/github.com/digitalocean/godo) + +You can view DigitalOcean API docs here: [https://developers.digitalocean.com/documentation/v2/](https://developers.digitalocean.com/documentation/v2/) + +## Install +```sh +go get github.com/digitalocean/godo@vX.Y.Z +``` + +where X.Y.Z is the [version](https://github.com/digitalocean/godo/releases) you need. + +or +```sh +go get github.com/digitalocean/godo +``` +for non Go modules usage or latest version. + +## Usage + +```go +import "github.com/digitalocean/godo" +``` + +Create a new DigitalOcean client, then use the exposed services to +access different parts of the DigitalOcean API. + +### Authentication + +Currently, Personal Access Token (PAT) is the only method of +authenticating with the API. You can manage your tokens +at the DigitalOcean Control Panel [Applications Page](https://cloud.digitalocean.com/settings/applications). + +You can then use your token to create a new client: + +```go +package main + +import ( + "github.com/digitalocean/godo" +) + +func main() { + client := godo.NewFromToken("my-digitalocean-api-token") +} +``` + +If you need to provide a `context.Context` to your new client, you should use [`godo.NewClient`](https://godoc.org/github.com/digitalocean/godo#NewClient) to manually construct a client instead. + +## Examples + + +To create a new Droplet: + +```go +dropletName := "super-cool-droplet" + +createRequest := &godo.DropletCreateRequest{ + Name: dropletName, + Region: "nyc3", + Size: "s-1vcpu-1gb", + Image: godo.DropletCreateImage{ + Slug: "ubuntu-14-04-x64", + }, +} + +ctx := context.TODO() + +newDroplet, _, err := client.Droplets.Create(ctx, createRequest) + +if err != nil { + fmt.Printf("Something bad happened: %s\n\n", err) + return err +} +``` + +### Pagination + +If a list of items is paginated by the API, you must request pages individually. For example, to fetch all Droplets: + +```go +func DropletList(ctx context.Context, client *godo.Client) ([]godo.Droplet, error) { + // create a list to hold our droplets + list := []godo.Droplet{} + + // create options. initially, these will be blank + opt := &godo.ListOptions{} + for { + droplets, resp, err := client.Droplets.List(ctx, opt) + if err != nil { + return nil, err + } + + // append the current page's droplets to our list + for _, d := range droplets { + list = append(list, d) + } + + // if we are at the last page, break out the for loop + if resp.Links == nil || resp.Links.IsLastPage() { + break + } + + page, err := resp.Links.CurrentPage() + if err != nil { + return nil, err + } + + // set the page we want for the next request + opt.Page = page + 1 + } + + return list, nil +} +``` + +## Versioning + +Each version of the client is tagged and the version is updated accordingly. + +To see the list of past versions, run `git tag`. + + +## Documentation + +For a comprehensive list of examples, check out the [API documentation](https://developers.digitalocean.com/documentation/v2/). + +For details on all the functionality in this library, see the [GoDoc](http://godoc.org/github.com/digitalocean/godo) documentation. + + +## Contributing + +We love pull requests! Please see the [contribution guidelines](CONTRIBUTING.md). diff --git a/vendor/github.com/digitalocean/godo/account.go b/vendor/github.com/digitalocean/godo/account.go new file mode 100644 index 0000000000000..7d3e105d34f6a --- /dev/null +++ b/vendor/github.com/digitalocean/godo/account.go @@ -0,0 +1,60 @@ +package godo + +import ( + "context" + "net/http" +) + +// AccountService is an interface for interfacing with the Account +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2/#account +type AccountService interface { + Get(context.Context) (*Account, *Response, error) +} + +// AccountServiceOp handles communication with the Account related methods of +// the DigitalOcean API. +type AccountServiceOp struct { + client *Client +} + +var _ AccountService = &AccountServiceOp{} + +// Account represents a DigitalOcean Account +type Account struct { + DropletLimit int `json:"droplet_limit,omitempty"` + FloatingIPLimit int `json:"floating_ip_limit,omitempty"` + VolumeLimit int `json:"volume_limit,omitempty"` + Email string `json:"email,omitempty"` + UUID string `json:"uuid,omitempty"` + EmailVerified bool `json:"email_verified,omitempty"` + Status string `json:"status,omitempty"` + StatusMessage string `json:"status_message,omitempty"` +} + +type accountRoot struct { + Account *Account `json:"account"` +} + +func (r Account) String() string { + return Stringify(r) +} + +// Get DigitalOcean account info +func (s *AccountServiceOp) Get(ctx context.Context) (*Account, *Response, error) { + + path := "v2/account" + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(accountRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Account, resp, err +} diff --git a/vendor/github.com/digitalocean/godo/action.go b/vendor/github.com/digitalocean/godo/action.go new file mode 100644 index 0000000000000..e3176005ba1df --- /dev/null +++ b/vendor/github.com/digitalocean/godo/action.go @@ -0,0 +1,108 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +const ( + actionsBasePath = "v2/actions" + + // ActionInProgress is an in progress action status + ActionInProgress = "in-progress" + + //ActionCompleted is a completed action status + ActionCompleted = "completed" +) + +// ActionsService handles communction with action related methods of the +// DigitalOcean API: https://developers.digitalocean.com/documentation/v2#actions +type ActionsService interface { + List(context.Context, *ListOptions) ([]Action, *Response, error) + Get(context.Context, int) (*Action, *Response, error) +} + +// ActionsServiceOp handles communition with the image action related methods of the +// DigitalOcean API. +type ActionsServiceOp struct { + client *Client +} + +var _ ActionsService = &ActionsServiceOp{} + +type actionsRoot struct { + Actions []Action `json:"actions"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +type actionRoot struct { + Event *Action `json:"action"` +} + +// Action represents a DigitalOcean Action +type Action struct { + ID int `json:"id"` + Status string `json:"status"` + Type string `json:"type"` + StartedAt *Timestamp `json:"started_at"` + CompletedAt *Timestamp `json:"completed_at"` + ResourceID int `json:"resource_id"` + ResourceType string `json:"resource_type"` + Region *Region `json:"region,omitempty"` + RegionSlug string `json:"region_slug,omitempty"` +} + +// List all actions +func (s *ActionsServiceOp) List(ctx context.Context, opt *ListOptions) ([]Action, *Response, error) { + path := actionsBasePath + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(actionsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Actions, resp, err +} + +// Get an action by ID. +func (s *ActionsServiceOp) Get(ctx context.Context, id int) (*Action, *Response, error) { + if id < 1 { + return nil, nil, NewArgError("id", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d", actionsBasePath, id) + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(actionRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Event, resp, err +} + +func (a Action) String() string { + return Stringify(a) +} diff --git a/vendor/github.com/digitalocean/godo/balance.go b/vendor/github.com/digitalocean/godo/balance.go new file mode 100644 index 0000000000000..4da69783613ab --- /dev/null +++ b/vendor/github.com/digitalocean/godo/balance.go @@ -0,0 +1,52 @@ +package godo + +import ( + "context" + "net/http" + "time" +) + +// BalanceService is an interface for interfacing with the Balance +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2/#balance +type BalanceService interface { + Get(context.Context) (*Balance, *Response, error) +} + +// BalanceServiceOp handles communication with the Balance related methods of +// the DigitalOcean API. +type BalanceServiceOp struct { + client *Client +} + +var _ BalanceService = &BalanceServiceOp{} + +// Balance represents a DigitalOcean Balance +type Balance struct { + MonthToDateBalance string `json:"month_to_date_balance"` + AccountBalance string `json:"account_balance"` + MonthToDateUsage string `json:"month_to_date_usage"` + GeneratedAt time.Time `json:"generated_at"` +} + +func (r Balance) String() string { + return Stringify(r) +} + +// Get DigitalOcean balance info +func (s *BalanceServiceOp) Get(ctx context.Context) (*Balance, *Response, error) { + path := "v2/customers/my/balance" + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(Balance) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root, resp, err +} diff --git a/vendor/github.com/digitalocean/godo/billing_history.go b/vendor/github.com/digitalocean/godo/billing_history.go new file mode 100644 index 0000000000000..a510100065712 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/billing_history.go @@ -0,0 +1,72 @@ +package godo + +import ( + "context" + "net/http" + "time" +) + +const billingHistoryBasePath = "v2/customers/my/billing_history" + +// BillingHistoryService is an interface for interfacing with the BillingHistory +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2/#billing_history +type BillingHistoryService interface { + List(context.Context, *ListOptions) (*BillingHistory, *Response, error) +} + +// BillingHistoryServiceOp handles communication with the BillingHistory related methods of +// the DigitalOcean API. +type BillingHistoryServiceOp struct { + client *Client +} + +var _ BillingHistoryService = &BillingHistoryServiceOp{} + +// BillingHistory represents a DigitalOcean Billing History +type BillingHistory struct { + BillingHistory []BillingHistoryEntry `json:"billing_history"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +// BillingHistoryEntry represents an entry in a customer's Billing History +type BillingHistoryEntry struct { + Description string `json:"description"` + Amount string `json:"amount"` + InvoiceID *string `json:"invoice_id"` + InvoiceUUID *string `json:"invoice_uuid"` + Date time.Time `json:"date"` + Type string `json:"type"` +} + +func (b BillingHistory) String() string { + return Stringify(b) +} + +// List the Billing History for a customer +func (s *BillingHistoryServiceOp) List(ctx context.Context, opt *ListOptions) (*BillingHistory, *Response, error) { + path, err := addOptions(billingHistoryBasePath, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(BillingHistory) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root, resp, err +} diff --git a/vendor/github.com/digitalocean/godo/cdn.go b/vendor/github.com/digitalocean/godo/cdn.go new file mode 100644 index 0000000000000..4c97d1174d6a9 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/cdn.go @@ -0,0 +1,218 @@ +package godo + +import ( + "context" + "fmt" + "net/http" + "time" +) + +const cdnBasePath = "v2/cdn/endpoints" + +// CDNService is an interface for managing Spaces CDN with the DigitalOcean API. +type CDNService interface { + List(context.Context, *ListOptions) ([]CDN, *Response, error) + Get(context.Context, string) (*CDN, *Response, error) + Create(context.Context, *CDNCreateRequest) (*CDN, *Response, error) + UpdateTTL(context.Context, string, *CDNUpdateTTLRequest) (*CDN, *Response, error) + UpdateCustomDomain(context.Context, string, *CDNUpdateCustomDomainRequest) (*CDN, *Response, error) + FlushCache(context.Context, string, *CDNFlushCacheRequest) (*Response, error) + Delete(context.Context, string) (*Response, error) +} + +// CDNServiceOp handles communication with the CDN related methods of the +// DigitalOcean API. +type CDNServiceOp struct { + client *Client +} + +var _ CDNService = &CDNServiceOp{} + +// CDN represents a DigitalOcean CDN +type CDN struct { + ID string `json:"id"` + Origin string `json:"origin"` + Endpoint string `json:"endpoint"` + CreatedAt time.Time `json:"created_at"` + TTL uint32 `json:"ttl"` + CertificateID string `json:"certificate_id,omitempty"` + CustomDomain string `json:"custom_domain,omitempty"` +} + +// CDNRoot represents a response from the DigitalOcean API +type cdnRoot struct { + Endpoint *CDN `json:"endpoint"` +} + +type cdnsRoot struct { + Endpoints []CDN `json:"endpoints"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +// CDNCreateRequest represents a request to create a CDN. +type CDNCreateRequest struct { + Origin string `json:"origin"` + TTL uint32 `json:"ttl"` + CustomDomain string `json:"custom_domain,omitempty"` + CertificateID string `json:"certificate_id,omitempty"` +} + +// CDNUpdateTTLRequest represents a request to update the ttl of a CDN. +type CDNUpdateTTLRequest struct { + TTL uint32 `json:"ttl"` +} + +// CDNUpdateCustomDomainRequest represents a request to update the custom domain of a CDN. +type CDNUpdateCustomDomainRequest struct { + CustomDomain string `json:"custom_domain"` + CertificateID string `json:"certificate_id"` +} + +// CDNFlushCacheRequest represents a request to flush cache of a CDN. +type CDNFlushCacheRequest struct { + Files []string `json:"files"` +} + +// List all CDN endpoints +func (c CDNServiceOp) List(ctx context.Context, opt *ListOptions) ([]CDN, *Response, error) { + path, err := addOptions(cdnBasePath, opt) + if err != nil { + return nil, nil, err + } + + req, err := c.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(cdnsRoot) + resp, err := c.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Endpoints, resp, err +} + +// Get individual CDN. It requires a non-empty cdn id. +func (c CDNServiceOp) Get(ctx context.Context, id string) (*CDN, *Response, error) { + if len(id) == 0 { + return nil, nil, NewArgError("id", "cannot be an empty string") + } + + path := fmt.Sprintf("%s/%s", cdnBasePath, id) + + req, err := c.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(cdnRoot) + resp, err := c.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Endpoint, resp, err +} + +// Create a new CDN +func (c CDNServiceOp) Create(ctx context.Context, createRequest *CDNCreateRequest) (*CDN, *Response, error) { + if createRequest == nil { + return nil, nil, NewArgError("createRequest", "cannot be nil") + } + + req, err := c.client.NewRequest(ctx, http.MethodPost, cdnBasePath, createRequest) + if err != nil { + return nil, nil, err + } + + root := new(cdnRoot) + resp, err := c.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Endpoint, resp, err +} + +// UpdateTTL updates the ttl of an individual CDN +func (c CDNServiceOp) UpdateTTL(ctx context.Context, id string, updateRequest *CDNUpdateTTLRequest) (*CDN, *Response, error) { + return c.update(ctx, id, updateRequest) +} + +// UpdateCustomDomain sets or removes the custom domain of an individual CDN +func (c CDNServiceOp) UpdateCustomDomain(ctx context.Context, id string, updateRequest *CDNUpdateCustomDomainRequest) (*CDN, *Response, error) { + return c.update(ctx, id, updateRequest) +} + +func (c CDNServiceOp) update(ctx context.Context, id string, updateRequest interface{}) (*CDN, *Response, error) { + if updateRequest == nil { + return nil, nil, NewArgError("updateRequest", "cannot be nil") + } + + if len(id) == 0 { + return nil, nil, NewArgError("id", "cannot be an empty string") + } + path := fmt.Sprintf("%s/%s", cdnBasePath, id) + + req, err := c.client.NewRequest(ctx, http.MethodPut, path, updateRequest) + if err != nil { + return nil, nil, err + } + + root := new(cdnRoot) + resp, err := c.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Endpoint, resp, err +} + +// FlushCache flushes the cache of an individual CDN. Requires a non-empty slice of file paths and/or wildcards +func (c CDNServiceOp) FlushCache(ctx context.Context, id string, flushCacheRequest *CDNFlushCacheRequest) (*Response, error) { + if flushCacheRequest == nil { + return nil, NewArgError("flushCacheRequest", "cannot be nil") + } + + if len(id) == 0 { + return nil, NewArgError("id", "cannot be an empty string") + } + + path := fmt.Sprintf("%s/%s/cache", cdnBasePath, id) + + req, err := c.client.NewRequest(ctx, http.MethodDelete, path, flushCacheRequest) + if err != nil { + return nil, err + } + + resp, err := c.client.Do(ctx, req, nil) + + return resp, err +} + +// Delete an individual CDN +func (c CDNServiceOp) Delete(ctx context.Context, id string) (*Response, error) { + if len(id) == 0 { + return nil, NewArgError("id", "cannot be an empty string") + } + + path := fmt.Sprintf("%s/%s", cdnBasePath, id) + + req, err := c.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + resp, err := c.client.Do(ctx, req, nil) + + return resp, err +} diff --git a/vendor/github.com/digitalocean/godo/certificates.go b/vendor/github.com/digitalocean/godo/certificates.go new file mode 100644 index 0000000000000..9a6bdb2daa0e4 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/certificates.go @@ -0,0 +1,130 @@ +package godo + +import ( + "context" + "net/http" + "path" +) + +const certificatesBasePath = "/v2/certificates" + +// CertificatesService is an interface for managing certificates with the DigitalOcean API. +// See: https://developers.digitalocean.com/documentation/v2/#certificates +type CertificatesService interface { + Get(context.Context, string) (*Certificate, *Response, error) + List(context.Context, *ListOptions) ([]Certificate, *Response, error) + Create(context.Context, *CertificateRequest) (*Certificate, *Response, error) + Delete(context.Context, string) (*Response, error) +} + +// Certificate represents a DigitalOcean certificate configuration. +type Certificate struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + DNSNames []string `json:"dns_names,omitempty"` + NotAfter string `json:"not_after,omitempty"` + SHA1Fingerprint string `json:"sha1_fingerprint,omitempty"` + Created string `json:"created_at,omitempty"` + State string `json:"state,omitempty"` + Type string `json:"type,omitempty"` +} + +// CertificateRequest represents configuration for a new certificate. +type CertificateRequest struct { + Name string `json:"name,omitempty"` + DNSNames []string `json:"dns_names,omitempty"` + PrivateKey string `json:"private_key,omitempty"` + LeafCertificate string `json:"leaf_certificate,omitempty"` + CertificateChain string `json:"certificate_chain,omitempty"` + Type string `json:"type,omitempty"` +} + +type certificateRoot struct { + Certificate *Certificate `json:"certificate"` +} + +type certificatesRoot struct { + Certificates []Certificate `json:"certificates"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +// CertificatesServiceOp handles communication with certificates methods of the DigitalOcean API. +type CertificatesServiceOp struct { + client *Client +} + +var _ CertificatesService = &CertificatesServiceOp{} + +// Get an existing certificate by its identifier. +func (c *CertificatesServiceOp) Get(ctx context.Context, cID string) (*Certificate, *Response, error) { + urlStr := path.Join(certificatesBasePath, cID) + + req, err := c.client.NewRequest(ctx, http.MethodGet, urlStr, nil) + if err != nil { + return nil, nil, err + } + + root := new(certificateRoot) + resp, err := c.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Certificate, resp, nil +} + +// List all certificates. +func (c *CertificatesServiceOp) List(ctx context.Context, opt *ListOptions) ([]Certificate, *Response, error) { + urlStr, err := addOptions(certificatesBasePath, opt) + if err != nil { + return nil, nil, err + } + + req, err := c.client.NewRequest(ctx, http.MethodGet, urlStr, nil) + if err != nil { + return nil, nil, err + } + + root := new(certificatesRoot) + resp, err := c.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Certificates, resp, nil +} + +// Create a new certificate with provided configuration. +func (c *CertificatesServiceOp) Create(ctx context.Context, cr *CertificateRequest) (*Certificate, *Response, error) { + req, err := c.client.NewRequest(ctx, http.MethodPost, certificatesBasePath, cr) + if err != nil { + return nil, nil, err + } + + root := new(certificateRoot) + resp, err := c.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Certificate, resp, nil +} + +// Delete a certificate by its identifier. +func (c *CertificatesServiceOp) Delete(ctx context.Context, cID string) (*Response, error) { + urlStr := path.Join(certificatesBasePath, cID) + + req, err := c.client.NewRequest(ctx, http.MethodDelete, urlStr, nil) + if err != nil { + return nil, err + } + + return c.client.Do(ctx, req, nil) +} diff --git a/vendor/github.com/digitalocean/godo/databases.go b/vendor/github.com/digitalocean/godo/databases.go new file mode 100644 index 0000000000000..42d83ee0c4178 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/databases.go @@ -0,0 +1,845 @@ +package godo + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" +) + +const ( + databaseBasePath = "/v2/databases" + databaseSinglePath = databaseBasePath + "/%s" + databaseResizePath = databaseBasePath + "/%s/resize" + databaseMigratePath = databaseBasePath + "/%s/migrate" + databaseMaintenancePath = databaseBasePath + "/%s/maintenance" + databaseBackupsPath = databaseBasePath + "/%s/backups" + databaseUsersPath = databaseBasePath + "/%s/users" + databaseUserPath = databaseBasePath + "/%s/users/%s" + databaseResetUserAuthPath = databaseUserPath + "/reset_auth" + databaseDBPath = databaseBasePath + "/%s/dbs/%s" + databaseDBsPath = databaseBasePath + "/%s/dbs" + databasePoolPath = databaseBasePath + "/%s/pools/%s" + databasePoolsPath = databaseBasePath + "/%s/pools" + databaseReplicaPath = databaseBasePath + "/%s/replicas/%s" + databaseReplicasPath = databaseBasePath + "/%s/replicas" + databaseEvictionPolicyPath = databaseBasePath + "/%s/eviction_policy" + databaseSQLModePath = databaseBasePath + "/%s/sql_mode" + databaseFirewallRulesPath = databaseBasePath + "/%s/firewall" +) + +// SQL Mode constants allow for MySQL-specific SQL flavor configuration. +const ( + SQLModeAllowInvalidDates = "ALLOW_INVALID_DATES" + SQLModeANSIQuotes = "ANSI_QUOTES" + SQLModeHighNotPrecedence = "HIGH_NOT_PRECEDENCE" + SQLModeIgnoreSpace = "IGNORE_SPACE" + SQLModeNoAuthCreateUser = "NO_AUTO_CREATE_USER" + SQLModeNoAutoValueOnZero = "NO_AUTO_VALUE_ON_ZERO" + SQLModeNoBackslashEscapes = "NO_BACKSLASH_ESCAPES" + SQLModeNoDirInCreate = "NO_DIR_IN_CREATE" + SQLModeNoEngineSubstitution = "NO_ENGINE_SUBSTITUTION" + SQLModeNoFieldOptions = "NO_FIELD_OPTIONS" + SQLModeNoKeyOptions = "NO_KEY_OPTIONS" + SQLModeNoTableOptions = "NO_TABLE_OPTIONS" + SQLModeNoUnsignedSubtraction = "NO_UNSIGNED_SUBTRACTION" + SQLModeNoZeroDate = "NO_ZERO_DATE" + SQLModeNoZeroInDate = "NO_ZERO_IN_DATE" + SQLModeOnlyFullGroupBy = "ONLY_FULL_GROUP_BY" + SQLModePadCharToFullLength = "PAD_CHAR_TO_FULL_LENGTH" + SQLModePipesAsConcat = "PIPES_AS_CONCAT" + SQLModeRealAsFloat = "REAL_AS_FLOAT" + SQLModeStrictAllTables = "STRICT_ALL_TABLES" + SQLModeStrictTransTables = "STRICT_TRANS_TABLES" + SQLModeANSI = "ANSI" + SQLModeDB2 = "DB2" + SQLModeMaxDB = "MAXDB" + SQLModeMSSQL = "MSSQL" + SQLModeMYSQL323 = "MYSQL323" + SQLModeMYSQL40 = "MYSQL40" + SQLModeOracle = "ORACLE" + SQLModePostgreSQL = "POSTGRESQL" + SQLModeTraditional = "TRADITIONAL" +) + +// SQL Auth constants allow for MySQL-specific user auth plugins +const ( + SQLAuthPluginNative = "mysql_native_password" + SQLAuthPluginCachingSHA2 = "caching_sha2_password" +) + +// Redis eviction policies supported by the managed Redis product. +const ( + EvictionPolicyNoEviction = "noeviction" + EvictionPolicyAllKeysLRU = "allkeys_lru" + EvictionPolicyAllKeysRandom = "allkeys_random" + EvictionPolicyVolatileLRU = "volatile_lru" + EvictionPolicyVolatileRandom = "volatile_random" + EvictionPolicyVolatileTTL = "volatile_ttl" +) + +// The DatabasesService provides access to the DigitalOcean managed database +// suite of products through the public API. Customers can create new database +// clusters, migrate them between regions, create replicas and interact with +// their configurations. Each database service is refered to as a Database. A +// SQL database service can have multiple databases residing in the system. To +// help make these entities distinct from Databases in godo, we refer to them +// here as DatabaseDBs. +// +// See: https://developers.digitalocean.com/documentation/v2#databases +type DatabasesService interface { + List(context.Context, *ListOptions) ([]Database, *Response, error) + Get(context.Context, string) (*Database, *Response, error) + Create(context.Context, *DatabaseCreateRequest) (*Database, *Response, error) + Delete(context.Context, string) (*Response, error) + Resize(context.Context, string, *DatabaseResizeRequest) (*Response, error) + Migrate(context.Context, string, *DatabaseMigrateRequest) (*Response, error) + UpdateMaintenance(context.Context, string, *DatabaseUpdateMaintenanceRequest) (*Response, error) + ListBackups(context.Context, string, *ListOptions) ([]DatabaseBackup, *Response, error) + GetUser(context.Context, string, string) (*DatabaseUser, *Response, error) + ListUsers(context.Context, string, *ListOptions) ([]DatabaseUser, *Response, error) + CreateUser(context.Context, string, *DatabaseCreateUserRequest) (*DatabaseUser, *Response, error) + DeleteUser(context.Context, string, string) (*Response, error) + ResetUserAuth(context.Context, string, string, *DatabaseResetUserAuthRequest) (*DatabaseUser, *Response, error) + ListDBs(context.Context, string, *ListOptions) ([]DatabaseDB, *Response, error) + CreateDB(context.Context, string, *DatabaseCreateDBRequest) (*DatabaseDB, *Response, error) + GetDB(context.Context, string, string) (*DatabaseDB, *Response, error) + DeleteDB(context.Context, string, string) (*Response, error) + ListPools(context.Context, string, *ListOptions) ([]DatabasePool, *Response, error) + CreatePool(context.Context, string, *DatabaseCreatePoolRequest) (*DatabasePool, *Response, error) + GetPool(context.Context, string, string) (*DatabasePool, *Response, error) + DeletePool(context.Context, string, string) (*Response, error) + GetReplica(context.Context, string, string) (*DatabaseReplica, *Response, error) + ListReplicas(context.Context, string, *ListOptions) ([]DatabaseReplica, *Response, error) + CreateReplica(context.Context, string, *DatabaseCreateReplicaRequest) (*DatabaseReplica, *Response, error) + DeleteReplica(context.Context, string, string) (*Response, error) + GetEvictionPolicy(context.Context, string) (string, *Response, error) + SetEvictionPolicy(context.Context, string, string) (*Response, error) + GetSQLMode(context.Context, string) (string, *Response, error) + SetSQLMode(context.Context, string, ...string) (*Response, error) + GetFirewallRules(context.Context, string) ([]DatabaseFirewallRule, *Response, error) + UpdateFirewallRules(context.Context, string, *DatabaseUpdateFirewallRulesRequest) (*Response, error) +} + +// DatabasesServiceOp handles communication with the Databases related methods +// of the DigitalOcean API. +type DatabasesServiceOp struct { + client *Client +} + +var _ DatabasesService = &DatabasesServiceOp{} + +// Database represents a DigitalOcean managed database product. These managed databases +// are usually comprised of a cluster of database nodes, a primary and 0 or more replicas. +// The EngineSlug is a string which indicates the type of database service. Some examples are +// "pg", "mysql" or "redis". A Database also includes connection information and other +// properties of the service like region, size and current status. +type Database struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + EngineSlug string `json:"engine,omitempty"` + VersionSlug string `json:"version,omitempty"` + Connection *DatabaseConnection `json:"connection,omitempty"` + PrivateConnection *DatabaseConnection `json:"private_connection,omitempty"` + Users []DatabaseUser `json:"users,omitempty"` + NumNodes int `json:"num_nodes,omitempty"` + SizeSlug string `json:"size,omitempty"` + DBNames []string `json:"db_names,omitempty"` + RegionSlug string `json:"region,omitempty"` + Status string `json:"status,omitempty"` + MaintenanceWindow *DatabaseMaintenanceWindow `json:"maintenance_window,omitempty"` + CreatedAt time.Time `json:"created_at,omitempty"` + PrivateNetworkUUID string `json:"private_network_uuid,omitempty"` + Tags []string `json:"tags,omitempty"` +} + +// DatabaseConnection represents a database connection +type DatabaseConnection struct { + URI string `json:"uri,omitempty"` + Database string `json:"database,omitempty"` + Host string `json:"host,omitempty"` + Port int `json:"port,omitempty"` + User string `json:"user,omitempty"` + Password string `json:"password,omitempty"` + SSL bool `json:"ssl,omitempty"` +} + +// DatabaseUser represents a user in the database +type DatabaseUser struct { + Name string `json:"name,omitempty"` + Role string `json:"role,omitempty"` + Password string `json:"password,omitempty"` + MySQLSettings *DatabaseMySQLUserSettings `json:"mysql_settings,omitempty"` +} + +// DatabaseMySQLUserSettings contains MySQL-specific user settings +type DatabaseMySQLUserSettings struct { + AuthPlugin string `json:"auth_plugin"` +} + +// DatabaseMaintenanceWindow represents the maintenance_window of a database +// cluster +type DatabaseMaintenanceWindow struct { + Day string `json:"day,omitempty"` + Hour string `json:"hour,omitempty"` + Pending bool `json:"pending,omitempty"` + Description []string `json:"description,omitempty"` +} + +// DatabaseBackup represents a database backup. +type DatabaseBackup struct { + CreatedAt time.Time `json:"created_at,omitempty"` + SizeGigabytes float64 `json:"size_gigabytes,omitempty"` +} + +// DatabaseCreateRequest represents a request to create a database cluster +type DatabaseCreateRequest struct { + Name string `json:"name,omitempty"` + EngineSlug string `json:"engine,omitempty"` + Version string `json:"version,omitempty"` + SizeSlug string `json:"size,omitempty"` + Region string `json:"region,omitempty"` + NumNodes int `json:"num_nodes,omitempty"` + PrivateNetworkUUID string `json:"private_network_uuid"` + Tags []string `json:"tags,omitempty"` +} + +// DatabaseResizeRequest can be used to initiate a database resize operation. +type DatabaseResizeRequest struct { + SizeSlug string `json:"size,omitempty"` + NumNodes int `json:"num_nodes,omitempty"` +} + +// DatabaseMigrateRequest can be used to initiate a database migrate operation. +type DatabaseMigrateRequest struct { + Region string `json:"region,omitempty"` + PrivateNetworkUUID string `json:"private_network_uuid"` +} + +// DatabaseUpdateMaintenanceRequest can be used to update the database's maintenance window. +type DatabaseUpdateMaintenanceRequest struct { + Day string `json:"day,omitempty"` + Hour string `json:"hour,omitempty"` +} + +// DatabaseDB represents an engine-specific database created within a database cluster. For SQL +// databases like PostgreSQL or MySQL, a "DB" refers to a database created on the RDBMS. For instance, +// a PostgreSQL database server can contain many database schemas, each with it's own settings, access +// permissions and data. ListDBs will return all databases present on the server. +type DatabaseDB struct { + Name string `json:"name"` +} + +// DatabaseReplica represents a read-only replica of a particular database +type DatabaseReplica struct { + Name string `json:"name"` + Connection *DatabaseConnection `json:"connection"` + PrivateConnection *DatabaseConnection `json:"private_connection,omitempty"` + Region string `json:"region"` + Status string `json:"status"` + CreatedAt time.Time `json:"created_at"` + PrivateNetworkUUID string `json:"private_network_uuid,omitempty"` + Tags []string `json:"tags,omitempty"` +} + +// DatabasePool represents a database connection pool +type DatabasePool struct { + User string `json:"user"` + Name string `json:"name"` + Size int `json:"size"` + Database string `json:"db"` + Mode string `json:"mode"` + Connection *DatabaseConnection `json:"connection"` + PrivateConnection *DatabaseConnection `json:"private_connection,omitempty"` +} + +// DatabaseCreatePoolRequest is used to create a new database connection pool +type DatabaseCreatePoolRequest struct { + User string `json:"user"` + Name string `json:"name"` + Size int `json:"size"` + Database string `json:"db"` + Mode string `json:"mode"` +} + +// DatabaseCreateUserRequest is used to create a new database user +type DatabaseCreateUserRequest struct { + Name string `json:"name"` + MySQLSettings *DatabaseMySQLUserSettings `json:"mysql_settings,omitempty"` +} + +// DatabaseResetUserAuth request is used to reset a users DB auth +type DatabaseResetUserAuthRequest struct { + MySQLSettings *DatabaseMySQLUserSettings `json:"mysql_settings,omitempty"` +} + +// DatabaseCreateDBRequest is used to create a new engine-specific database within the cluster +type DatabaseCreateDBRequest struct { + Name string `json:"name"` +} + +// DatabaseCreateReplicaRequest is used to create a new read-only replica +type DatabaseCreateReplicaRequest struct { + Name string `json:"name"` + Region string `json:"region"` + Size string `json:"size"` + PrivateNetworkUUID string `json:"private_network_uuid"` + Tags []string `json:"tags,omitempty"` +} + +// DatabaseUpdateFirewallRulesRequest is used to set the firewall rules for a database +type DatabaseUpdateFirewallRulesRequest struct { + Rules []*DatabaseFirewallRule `json:"rules"` +} + +// DatabaseFirewallRule is a rule describing an inbound source to a database +type DatabaseFirewallRule struct { + UUID string `json:"uuid"` + ClusterUUID string `json:"cluster_uuid"` + Type string `json:"type"` + Value string `json:"value"` + CreatedAt time.Time `json:"created_at"` +} + +type databaseUserRoot struct { + User *DatabaseUser `json:"user"` +} + +type databaseUsersRoot struct { + Users []DatabaseUser `json:"users"` +} + +type databaseDBRoot struct { + DB *DatabaseDB `json:"db"` +} + +type databaseDBsRoot struct { + DBs []DatabaseDB `json:"dbs"` +} + +type databasesRoot struct { + Databases []Database `json:"databases"` +} + +type databaseRoot struct { + Database *Database `json:"database"` +} + +type databaseBackupsRoot struct { + Backups []DatabaseBackup `json:"backups"` +} + +type databasePoolRoot struct { + Pool *DatabasePool `json:"pool"` +} + +type databasePoolsRoot struct { + Pools []DatabasePool `json:"pools"` +} + +type databaseReplicaRoot struct { + Replica *DatabaseReplica `json:"replica"` +} + +type databaseReplicasRoot struct { + Replicas []DatabaseReplica `json:"replicas"` +} + +type evictionPolicyRoot struct { + EvictionPolicy string `json:"eviction_policy"` +} + +type sqlModeRoot struct { + SQLMode string `json:"sql_mode"` +} + +type databaseFirewallRuleRoot struct { + Rules []DatabaseFirewallRule `json:"rules"` +} + +// URN returns a URN identifier for the database +func (d Database) URN() string { + return ToURN("dbaas", d.ID) +} + +// List returns a list of the Databases visible with the caller's API token +func (svc *DatabasesServiceOp) List(ctx context.Context, opts *ListOptions) ([]Database, *Response, error) { + path := databaseBasePath + path, err := addOptions(path, opts) + if err != nil { + return nil, nil, err + } + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databasesRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Databases, resp, nil +} + +// Get retrieves the details of a database cluster +func (svc *DatabasesServiceOp) Get(ctx context.Context, databaseID string) (*Database, *Response, error) { + path := fmt.Sprintf(databaseSinglePath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databaseRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Database, resp, nil +} + +// Create creates a database cluster +func (svc *DatabasesServiceOp) Create(ctx context.Context, create *DatabaseCreateRequest) (*Database, *Response, error) { + path := databaseBasePath + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, create) + if err != nil { + return nil, nil, err + } + root := new(databaseRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Database, resp, nil +} + +// Delete deletes a database cluster. There is no way to recover a cluster once +// it has been destroyed. +func (svc *DatabasesServiceOp) Delete(ctx context.Context, databaseID string) (*Response, error) { + path := fmt.Sprintf("%s/%s", databaseBasePath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// Resize resizes a database cluster by number of nodes or size +func (svc *DatabasesServiceOp) Resize(ctx context.Context, databaseID string, resize *DatabaseResizeRequest) (*Response, error) { + path := fmt.Sprintf(databaseResizePath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodPut, path, resize) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// Migrate migrates a database cluster to a new region +func (svc *DatabasesServiceOp) Migrate(ctx context.Context, databaseID string, migrate *DatabaseMigrateRequest) (*Response, error) { + path := fmt.Sprintf(databaseMigratePath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodPut, path, migrate) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// UpdateMaintenance updates the maintenance window on a cluster +func (svc *DatabasesServiceOp) UpdateMaintenance(ctx context.Context, databaseID string, maintenance *DatabaseUpdateMaintenanceRequest) (*Response, error) { + path := fmt.Sprintf(databaseMaintenancePath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodPut, path, maintenance) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// ListBackups returns a list of the current backups of a database +func (svc *DatabasesServiceOp) ListBackups(ctx context.Context, databaseID string, opts *ListOptions) ([]DatabaseBackup, *Response, error) { + path := fmt.Sprintf(databaseBackupsPath, databaseID) + path, err := addOptions(path, opts) + if err != nil { + return nil, nil, err + } + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databaseBackupsRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Backups, resp, nil +} + +// GetUser returns the database user identified by userID +func (svc *DatabasesServiceOp) GetUser(ctx context.Context, databaseID, userID string) (*DatabaseUser, *Response, error) { + path := fmt.Sprintf(databaseUserPath, databaseID, userID) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databaseUserRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.User, resp, nil +} + +// ListUsers returns all database users for the database +func (svc *DatabasesServiceOp) ListUsers(ctx context.Context, databaseID string, opts *ListOptions) ([]DatabaseUser, *Response, error) { + path := fmt.Sprintf(databaseUsersPath, databaseID) + path, err := addOptions(path, opts) + if err != nil { + return nil, nil, err + } + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databaseUsersRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Users, resp, nil +} + +// CreateUser will create a new database user +func (svc *DatabasesServiceOp) CreateUser(ctx context.Context, databaseID string, createUser *DatabaseCreateUserRequest) (*DatabaseUser, *Response, error) { + path := fmt.Sprintf(databaseUsersPath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, createUser) + if err != nil { + return nil, nil, err + } + root := new(databaseUserRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.User, resp, nil +} + +func (svc *DatabasesServiceOp) ResetUserAuth(ctx context.Context, databaseID, userID string, resetAuth *DatabaseResetUserAuthRequest) (*DatabaseUser, *Response, error) { + path := fmt.Sprintf(databaseResetUserAuthPath, databaseID, userID) + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, resetAuth) + if err != nil { + return nil, nil, err + } + root := new(databaseUserRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.User, resp, nil +} + +// DeleteUser will delete an existing database user +func (svc *DatabasesServiceOp) DeleteUser(ctx context.Context, databaseID, userID string) (*Response, error) { + path := fmt.Sprintf(databaseUserPath, databaseID, userID) + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// ListDBs returns all databases for a given database cluster +func (svc *DatabasesServiceOp) ListDBs(ctx context.Context, databaseID string, opts *ListOptions) ([]DatabaseDB, *Response, error) { + path := fmt.Sprintf(databaseDBsPath, databaseID) + path, err := addOptions(path, opts) + if err != nil { + return nil, nil, err + } + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databaseDBsRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.DBs, resp, nil +} + +// GetDB returns a single database by name +func (svc *DatabasesServiceOp) GetDB(ctx context.Context, databaseID, name string) (*DatabaseDB, *Response, error) { + path := fmt.Sprintf(databaseDBPath, databaseID, name) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databaseDBRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.DB, resp, nil +} + +// CreateDB will create a new database +func (svc *DatabasesServiceOp) CreateDB(ctx context.Context, databaseID string, createDB *DatabaseCreateDBRequest) (*DatabaseDB, *Response, error) { + path := fmt.Sprintf(databaseDBsPath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, createDB) + if err != nil { + return nil, nil, err + } + root := new(databaseDBRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.DB, resp, nil +} + +// DeleteDB will delete an existing database +func (svc *DatabasesServiceOp) DeleteDB(ctx context.Context, databaseID, name string) (*Response, error) { + path := fmt.Sprintf(databaseDBPath, databaseID, name) + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// ListPools returns all connection pools for a given database cluster +func (svc *DatabasesServiceOp) ListPools(ctx context.Context, databaseID string, opts *ListOptions) ([]DatabasePool, *Response, error) { + path := fmt.Sprintf(databasePoolsPath, databaseID) + path, err := addOptions(path, opts) + if err != nil { + return nil, nil, err + } + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databasePoolsRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Pools, resp, nil +} + +// GetPool returns a single database connection pool by name +func (svc *DatabasesServiceOp) GetPool(ctx context.Context, databaseID, name string) (*DatabasePool, *Response, error) { + path := fmt.Sprintf(databasePoolPath, databaseID, name) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databasePoolRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Pool, resp, nil +} + +// CreatePool will create a new database connection pool +func (svc *DatabasesServiceOp) CreatePool(ctx context.Context, databaseID string, createPool *DatabaseCreatePoolRequest) (*DatabasePool, *Response, error) { + path := fmt.Sprintf(databasePoolsPath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, createPool) + if err != nil { + return nil, nil, err + } + root := new(databasePoolRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Pool, resp, nil +} + +// DeletePool will delete an existing database connection pool +func (svc *DatabasesServiceOp) DeletePool(ctx context.Context, databaseID, name string) (*Response, error) { + path := fmt.Sprintf(databasePoolPath, databaseID, name) + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// GetReplica returns a single database replica +func (svc *DatabasesServiceOp) GetReplica(ctx context.Context, databaseID, name string) (*DatabaseReplica, *Response, error) { + path := fmt.Sprintf(databaseReplicaPath, databaseID, name) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databaseReplicaRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Replica, resp, nil +} + +// ListReplicas returns all read-only replicas for a given database cluster +func (svc *DatabasesServiceOp) ListReplicas(ctx context.Context, databaseID string, opts *ListOptions) ([]DatabaseReplica, *Response, error) { + path := fmt.Sprintf(databaseReplicasPath, databaseID) + path, err := addOptions(path, opts) + if err != nil { + return nil, nil, err + } + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databaseReplicasRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Replicas, resp, nil +} + +// CreateReplica will create a new database connection pool +func (svc *DatabasesServiceOp) CreateReplica(ctx context.Context, databaseID string, createReplica *DatabaseCreateReplicaRequest) (*DatabaseReplica, *Response, error) { + path := fmt.Sprintf(databaseReplicasPath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, createReplica) + if err != nil { + return nil, nil, err + } + root := new(databaseReplicaRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Replica, resp, nil +} + +// DeleteReplica will delete an existing database replica +func (svc *DatabasesServiceOp) DeleteReplica(ctx context.Context, databaseID, name string) (*Response, error) { + path := fmt.Sprintf(databaseReplicaPath, databaseID, name) + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// GetEvictionPolicy loads the eviction policy for a given Redis cluster. +func (svc *DatabasesServiceOp) GetEvictionPolicy(ctx context.Context, databaseID string) (string, *Response, error) { + path := fmt.Sprintf(databaseEvictionPolicyPath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return "", nil, err + } + root := new(evictionPolicyRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return "", resp, err + } + return root.EvictionPolicy, resp, nil +} + +// SetEvictionPolicy updates the eviction policy for a given Redis cluster. +// +// The valid eviction policies are documented by the exported string constants +// with the prefix `EvictionPolicy`. +func (svc *DatabasesServiceOp) SetEvictionPolicy(ctx context.Context, databaseID, policy string) (*Response, error) { + path := fmt.Sprintf(databaseEvictionPolicyPath, databaseID) + root := &evictionPolicyRoot{EvictionPolicy: policy} + req, err := svc.client.NewRequest(ctx, http.MethodPut, path, root) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// GetSQLMode loads the SQL Mode settings for a given MySQL cluster. +func (svc *DatabasesServiceOp) GetSQLMode(ctx context.Context, databaseID string) (string, *Response, error) { + path := fmt.Sprintf(databaseSQLModePath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return "", nil, err + } + root := &sqlModeRoot{} + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return "", resp, err + } + return root.SQLMode, resp, nil +} + +// SetSQLMode updates the SQL Mode settings for a given MySQL cluster. +func (svc *DatabasesServiceOp) SetSQLMode(ctx context.Context, databaseID string, sqlModes ...string) (*Response, error) { + path := fmt.Sprintf(databaseSQLModePath, databaseID) + root := &sqlModeRoot{SQLMode: strings.Join(sqlModes, ",")} + req, err := svc.client.NewRequest(ctx, http.MethodPut, path, root) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// GetFirewallRules loads the inbound sources for a given cluster. +func (svc *DatabasesServiceOp) GetFirewallRules(ctx context.Context, databaseID string) ([]DatabaseFirewallRule, *Response, error) { + path := fmt.Sprintf(databaseFirewallRulesPath, databaseID) + root := new(databaseFirewallRuleRoot) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Rules, resp, nil +} + +// UpdateFirewallRules sets the inbound sources for a given cluster. +func (svc *DatabasesServiceOp) UpdateFirewallRules(ctx context.Context, databaseID string, firewallRulesReq *DatabaseUpdateFirewallRulesRequest) (*Response, error) { + path := fmt.Sprintf(databaseFirewallRulesPath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodPut, path, firewallRulesReq) + if err != nil { + return nil, err + } + return svc.client.Do(ctx, req, nil) +} diff --git a/vendor/github.com/digitalocean/godo/doc.go b/vendor/github.com/digitalocean/godo/doc.go new file mode 100644 index 0000000000000..ac812e97407cc --- /dev/null +++ b/vendor/github.com/digitalocean/godo/doc.go @@ -0,0 +1,2 @@ +// Package godo is the DigtalOcean API v2 client for Go. +package godo diff --git a/vendor/github.com/digitalocean/godo/domains.go b/vendor/github.com/digitalocean/godo/domains.go new file mode 100644 index 0000000000000..43c04244738c8 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/domains.go @@ -0,0 +1,341 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +const domainsBasePath = "v2/domains" + +// DomainsService is an interface for managing DNS with the DigitalOcean API. +// See: https://developers.digitalocean.com/documentation/v2#domains and +// https://developers.digitalocean.com/documentation/v2#domain-records +type DomainsService interface { + List(context.Context, *ListOptions) ([]Domain, *Response, error) + Get(context.Context, string) (*Domain, *Response, error) + Create(context.Context, *DomainCreateRequest) (*Domain, *Response, error) + Delete(context.Context, string) (*Response, error) + + Records(context.Context, string, *ListOptions) ([]DomainRecord, *Response, error) + Record(context.Context, string, int) (*DomainRecord, *Response, error) + DeleteRecord(context.Context, string, int) (*Response, error) + EditRecord(context.Context, string, int, *DomainRecordEditRequest) (*DomainRecord, *Response, error) + CreateRecord(context.Context, string, *DomainRecordEditRequest) (*DomainRecord, *Response, error) +} + +// DomainsServiceOp handles communication with the domain related methods of the +// DigitalOcean API. +type DomainsServiceOp struct { + client *Client +} + +var _ DomainsService = &DomainsServiceOp{} + +// Domain represents a DigitalOcean domain +type Domain struct { + Name string `json:"name"` + TTL int `json:"ttl"` + ZoneFile string `json:"zone_file"` +} + +// domainRoot represents a response from the DigitalOcean API +type domainRoot struct { + Domain *Domain `json:"domain"` +} + +type domainsRoot struct { + Domains []Domain `json:"domains"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +// DomainCreateRequest respresents a request to create a domain. +type DomainCreateRequest struct { + Name string `json:"name"` + IPAddress string `json:"ip_address,omitempty"` +} + +// DomainRecordRoot is the root of an individual Domain Record response +type domainRecordRoot struct { + DomainRecord *DomainRecord `json:"domain_record"` +} + +// DomainRecordsRoot is the root of a group of Domain Record responses +type domainRecordsRoot struct { + DomainRecords []DomainRecord `json:"domain_records"` + Links *Links `json:"links"` +} + +// DomainRecord represents a DigitalOcean DomainRecord +type DomainRecord struct { + ID int `json:"id,float64,omitempty"` + Type string `json:"type,omitempty"` + Name string `json:"name,omitempty"` + Data string `json:"data,omitempty"` + Priority int `json:"priority"` + Port int `json:"port,omitempty"` + TTL int `json:"ttl,omitempty"` + Weight int `json:"weight"` + Flags int `json:"flags"` + Tag string `json:"tag,omitempty"` +} + +// DomainRecordEditRequest represents a request to update a domain record. +type DomainRecordEditRequest struct { + Type string `json:"type,omitempty"` + Name string `json:"name,omitempty"` + Data string `json:"data,omitempty"` + Priority int `json:"priority"` + Port int `json:"port,omitempty"` + TTL int `json:"ttl,omitempty"` + Weight int `json:"weight"` + Flags int `json:"flags"` + Tag string `json:"tag,omitempty"` +} + +func (d Domain) String() string { + return Stringify(d) +} + +func (d Domain) URN() string { + return ToURN("Domain", d.Name) +} + +// List all domains. +func (s DomainsServiceOp) List(ctx context.Context, opt *ListOptions) ([]Domain, *Response, error) { + path := domainsBasePath + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(domainsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Domains, resp, err +} + +// Get individual domain. It requires a non-empty domain name. +func (s *DomainsServiceOp) Get(ctx context.Context, name string) (*Domain, *Response, error) { + if len(name) < 1 { + return nil, nil, NewArgError("name", "cannot be an empty string") + } + + path := fmt.Sprintf("%s/%s", domainsBasePath, name) + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(domainRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Domain, resp, err +} + +// Create a new domain +func (s *DomainsServiceOp) Create(ctx context.Context, createRequest *DomainCreateRequest) (*Domain, *Response, error) { + if createRequest == nil { + return nil, nil, NewArgError("createRequest", "cannot be nil") + } + + path := domainsBasePath + + req, err := s.client.NewRequest(ctx, http.MethodPost, path, createRequest) + if err != nil { + return nil, nil, err + } + + root := new(domainRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Domain, resp, err +} + +// Delete domain +func (s *DomainsServiceOp) Delete(ctx context.Context, name string) (*Response, error) { + if len(name) < 1 { + return nil, NewArgError("name", "cannot be an empty string") + } + + path := fmt.Sprintf("%s/%s", domainsBasePath, name) + + req, err := s.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + resp, err := s.client.Do(ctx, req, nil) + + return resp, err +} + +// Converts a DomainRecord to a string. +func (d DomainRecord) String() string { + return Stringify(d) +} + +// Converts a DomainRecordEditRequest to a string. +func (d DomainRecordEditRequest) String() string { + return Stringify(d) +} + +// Records returns a slice of DomainRecords for a domain +func (s *DomainsServiceOp) Records(ctx context.Context, domain string, opt *ListOptions) ([]DomainRecord, *Response, error) { + if len(domain) < 1 { + return nil, nil, NewArgError("domain", "cannot be an empty string") + } + + path := fmt.Sprintf("%s/%s/records", domainsBasePath, domain) + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(domainRecordsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + + return root.DomainRecords, resp, err +} + +// Record returns the record id from a domain +func (s *DomainsServiceOp) Record(ctx context.Context, domain string, id int) (*DomainRecord, *Response, error) { + if len(domain) < 1 { + return nil, nil, NewArgError("domain", "cannot be an empty string") + } + + if id < 1 { + return nil, nil, NewArgError("id", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%s/records/%d", domainsBasePath, domain, id) + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + record := new(domainRecordRoot) + resp, err := s.client.Do(ctx, req, record) + if err != nil { + return nil, resp, err + } + + return record.DomainRecord, resp, err +} + +// DeleteRecord deletes a record from a domain identified by id +func (s *DomainsServiceOp) DeleteRecord(ctx context.Context, domain string, id int) (*Response, error) { + if len(domain) < 1 { + return nil, NewArgError("domain", "cannot be an empty string") + } + + if id < 1 { + return nil, NewArgError("id", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%s/records/%d", domainsBasePath, domain, id) + + req, err := s.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + resp, err := s.client.Do(ctx, req, nil) + + return resp, err +} + +// EditRecord edits a record using a DomainRecordEditRequest +func (s *DomainsServiceOp) EditRecord(ctx context.Context, + domain string, + id int, + editRequest *DomainRecordEditRequest, +) (*DomainRecord, *Response, error) { + if len(domain) < 1 { + return nil, nil, NewArgError("domain", "cannot be an empty string") + } + + if id < 1 { + return nil, nil, NewArgError("id", "cannot be less than 1") + } + + if editRequest == nil { + return nil, nil, NewArgError("editRequest", "cannot be nil") + } + + path := fmt.Sprintf("%s/%s/records/%d", domainsBasePath, domain, id) + + req, err := s.client.NewRequest(ctx, http.MethodPut, path, editRequest) + if err != nil { + return nil, nil, err + } + + root := new(domainRecordRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.DomainRecord, resp, err +} + +// CreateRecord creates a record using a DomainRecordEditRequest +func (s *DomainsServiceOp) CreateRecord(ctx context.Context, + domain string, + createRequest *DomainRecordEditRequest) (*DomainRecord, *Response, error) { + if len(domain) < 1 { + return nil, nil, NewArgError("domain", "cannot be empty string") + } + + if createRequest == nil { + return nil, nil, NewArgError("createRequest", "cannot be nil") + } + + path := fmt.Sprintf("%s/%s/records", domainsBasePath, domain) + req, err := s.client.NewRequest(ctx, http.MethodPost, path, createRequest) + + if err != nil { + return nil, nil, err + } + + d := new(domainRecordRoot) + resp, err := s.client.Do(ctx, req, d) + if err != nil { + return nil, resp, err + } + + return d.DomainRecord, resp, err +} diff --git a/vendor/github.com/digitalocean/godo/droplet_actions.go b/vendor/github.com/digitalocean/godo/droplet_actions.go new file mode 100644 index 0000000000000..ddeacfc862493 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/droplet_actions.go @@ -0,0 +1,329 @@ +package godo + +import ( + "context" + "fmt" + "net/http" + "net/url" +) + +// ActionRequest reprents DigitalOcean Action Request +type ActionRequest map[string]interface{} + +// DropletActionsService is an interface for interfacing with the Droplet actions +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2#droplet-actions +type DropletActionsService interface { + Shutdown(context.Context, int) (*Action, *Response, error) + ShutdownByTag(context.Context, string) ([]Action, *Response, error) + PowerOff(context.Context, int) (*Action, *Response, error) + PowerOffByTag(context.Context, string) ([]Action, *Response, error) + PowerOn(context.Context, int) (*Action, *Response, error) + PowerOnByTag(context.Context, string) ([]Action, *Response, error) + PowerCycle(context.Context, int) (*Action, *Response, error) + PowerCycleByTag(context.Context, string) ([]Action, *Response, error) + Reboot(context.Context, int) (*Action, *Response, error) + Restore(context.Context, int, int) (*Action, *Response, error) + Resize(context.Context, int, string, bool) (*Action, *Response, error) + Rename(context.Context, int, string) (*Action, *Response, error) + Snapshot(context.Context, int, string) (*Action, *Response, error) + SnapshotByTag(context.Context, string, string) ([]Action, *Response, error) + EnableBackups(context.Context, int) (*Action, *Response, error) + EnableBackupsByTag(context.Context, string) ([]Action, *Response, error) + DisableBackups(context.Context, int) (*Action, *Response, error) + DisableBackupsByTag(context.Context, string) ([]Action, *Response, error) + PasswordReset(context.Context, int) (*Action, *Response, error) + RebuildByImageID(context.Context, int, int) (*Action, *Response, error) + RebuildByImageSlug(context.Context, int, string) (*Action, *Response, error) + ChangeKernel(context.Context, int, int) (*Action, *Response, error) + EnableIPv6(context.Context, int) (*Action, *Response, error) + EnableIPv6ByTag(context.Context, string) ([]Action, *Response, error) + EnablePrivateNetworking(context.Context, int) (*Action, *Response, error) + EnablePrivateNetworkingByTag(context.Context, string) ([]Action, *Response, error) + Get(context.Context, int, int) (*Action, *Response, error) + GetByURI(context.Context, string) (*Action, *Response, error) +} + +// DropletActionsServiceOp handles communication with the Droplet action related +// methods of the DigitalOcean API. +type DropletActionsServiceOp struct { + client *Client +} + +var _ DropletActionsService = &DropletActionsServiceOp{} + +// Shutdown a Droplet +func (s *DropletActionsServiceOp) Shutdown(ctx context.Context, id int) (*Action, *Response, error) { + request := &ActionRequest{"type": "shutdown"} + return s.doAction(ctx, id, request) +} + +// ShutdownByTag shuts down Droplets matched by a Tag. +func (s *DropletActionsServiceOp) ShutdownByTag(ctx context.Context, tag string) ([]Action, *Response, error) { + request := &ActionRequest{"type": "shutdown"} + return s.doActionByTag(ctx, tag, request) +} + +// PowerOff a Droplet +func (s *DropletActionsServiceOp) PowerOff(ctx context.Context, id int) (*Action, *Response, error) { + request := &ActionRequest{"type": "power_off"} + return s.doAction(ctx, id, request) +} + +// PowerOffByTag powers off Droplets matched by a Tag. +func (s *DropletActionsServiceOp) PowerOffByTag(ctx context.Context, tag string) ([]Action, *Response, error) { + request := &ActionRequest{"type": "power_off"} + return s.doActionByTag(ctx, tag, request) +} + +// PowerOn a Droplet +func (s *DropletActionsServiceOp) PowerOn(ctx context.Context, id int) (*Action, *Response, error) { + request := &ActionRequest{"type": "power_on"} + return s.doAction(ctx, id, request) +} + +// PowerOnByTag powers on Droplets matched by a Tag. +func (s *DropletActionsServiceOp) PowerOnByTag(ctx context.Context, tag string) ([]Action, *Response, error) { + request := &ActionRequest{"type": "power_on"} + return s.doActionByTag(ctx, tag, request) +} + +// PowerCycle a Droplet +func (s *DropletActionsServiceOp) PowerCycle(ctx context.Context, id int) (*Action, *Response, error) { + request := &ActionRequest{"type": "power_cycle"} + return s.doAction(ctx, id, request) +} + +// PowerCycleByTag power cycles Droplets matched by a Tag. +func (s *DropletActionsServiceOp) PowerCycleByTag(ctx context.Context, tag string) ([]Action, *Response, error) { + request := &ActionRequest{"type": "power_cycle"} + return s.doActionByTag(ctx, tag, request) +} + +// Reboot a Droplet +func (s *DropletActionsServiceOp) Reboot(ctx context.Context, id int) (*Action, *Response, error) { + request := &ActionRequest{"type": "reboot"} + return s.doAction(ctx, id, request) +} + +// Restore an image to a Droplet +func (s *DropletActionsServiceOp) Restore(ctx context.Context, id, imageID int) (*Action, *Response, error) { + requestType := "restore" + request := &ActionRequest{ + "type": requestType, + "image": float64(imageID), + } + return s.doAction(ctx, id, request) +} + +// Resize a Droplet +func (s *DropletActionsServiceOp) Resize(ctx context.Context, id int, sizeSlug string, resizeDisk bool) (*Action, *Response, error) { + requestType := "resize" + request := &ActionRequest{ + "type": requestType, + "size": sizeSlug, + "disk": resizeDisk, + } + return s.doAction(ctx, id, request) +} + +// Rename a Droplet +func (s *DropletActionsServiceOp) Rename(ctx context.Context, id int, name string) (*Action, *Response, error) { + requestType := "rename" + request := &ActionRequest{ + "type": requestType, + "name": name, + } + return s.doAction(ctx, id, request) +} + +// Snapshot a Droplet. +func (s *DropletActionsServiceOp) Snapshot(ctx context.Context, id int, name string) (*Action, *Response, error) { + requestType := "snapshot" + request := &ActionRequest{ + "type": requestType, + "name": name, + } + return s.doAction(ctx, id, request) +} + +// SnapshotByTag snapshots Droplets matched by a Tag. +func (s *DropletActionsServiceOp) SnapshotByTag(ctx context.Context, tag string, name string) ([]Action, *Response, error) { + requestType := "snapshot" + request := &ActionRequest{ + "type": requestType, + "name": name, + } + return s.doActionByTag(ctx, tag, request) +} + +// EnableBackups enables backups for a Droplet. +func (s *DropletActionsServiceOp) EnableBackups(ctx context.Context, id int) (*Action, *Response, error) { + request := &ActionRequest{"type": "enable_backups"} + return s.doAction(ctx, id, request) +} + +// EnableBackupsByTag enables backups for Droplets matched by a Tag. +func (s *DropletActionsServiceOp) EnableBackupsByTag(ctx context.Context, tag string) ([]Action, *Response, error) { + request := &ActionRequest{"type": "enable_backups"} + return s.doActionByTag(ctx, tag, request) +} + +// DisableBackups disables backups for a Droplet. +func (s *DropletActionsServiceOp) DisableBackups(ctx context.Context, id int) (*Action, *Response, error) { + request := &ActionRequest{"type": "disable_backups"} + return s.doAction(ctx, id, request) +} + +// DisableBackupsByTag disables backups for Droplet matched by a Tag. +func (s *DropletActionsServiceOp) DisableBackupsByTag(ctx context.Context, tag string) ([]Action, *Response, error) { + request := &ActionRequest{"type": "disable_backups"} + return s.doActionByTag(ctx, tag, request) +} + +// PasswordReset resets the password for a Droplet. +func (s *DropletActionsServiceOp) PasswordReset(ctx context.Context, id int) (*Action, *Response, error) { + request := &ActionRequest{"type": "password_reset"} + return s.doAction(ctx, id, request) +} + +// RebuildByImageID rebuilds a Droplet from an image with a given id. +func (s *DropletActionsServiceOp) RebuildByImageID(ctx context.Context, id, imageID int) (*Action, *Response, error) { + request := &ActionRequest{"type": "rebuild", "image": imageID} + return s.doAction(ctx, id, request) +} + +// RebuildByImageSlug rebuilds a Droplet from an Image matched by a given Slug. +func (s *DropletActionsServiceOp) RebuildByImageSlug(ctx context.Context, id int, slug string) (*Action, *Response, error) { + request := &ActionRequest{"type": "rebuild", "image": slug} + return s.doAction(ctx, id, request) +} + +// ChangeKernel changes the kernel for a Droplet. +func (s *DropletActionsServiceOp) ChangeKernel(ctx context.Context, id, kernelID int) (*Action, *Response, error) { + request := &ActionRequest{"type": "change_kernel", "kernel": kernelID} + return s.doAction(ctx, id, request) +} + +// EnableIPv6 enables IPv6 for a Droplet. +func (s *DropletActionsServiceOp) EnableIPv6(ctx context.Context, id int) (*Action, *Response, error) { + request := &ActionRequest{"type": "enable_ipv6"} + return s.doAction(ctx, id, request) +} + +// EnableIPv6ByTag enables IPv6 for Droplets matched by a Tag. +func (s *DropletActionsServiceOp) EnableIPv6ByTag(ctx context.Context, tag string) ([]Action, *Response, error) { + request := &ActionRequest{"type": "enable_ipv6"} + return s.doActionByTag(ctx, tag, request) +} + +// EnablePrivateNetworking enables private networking for a Droplet. +func (s *DropletActionsServiceOp) EnablePrivateNetworking(ctx context.Context, id int) (*Action, *Response, error) { + request := &ActionRequest{"type": "enable_private_networking"} + return s.doAction(ctx, id, request) +} + +// EnablePrivateNetworkingByTag enables private networking for Droplets matched by a Tag. +func (s *DropletActionsServiceOp) EnablePrivateNetworkingByTag(ctx context.Context, tag string) ([]Action, *Response, error) { + request := &ActionRequest{"type": "enable_private_networking"} + return s.doActionByTag(ctx, tag, request) +} + +func (s *DropletActionsServiceOp) doAction(ctx context.Context, id int, request *ActionRequest) (*Action, *Response, error) { + if id < 1 { + return nil, nil, NewArgError("id", "cannot be less than 1") + } + + if request == nil { + return nil, nil, NewArgError("request", "request can't be nil") + } + + path := dropletActionPath(id) + + req, err := s.client.NewRequest(ctx, http.MethodPost, path, request) + if err != nil { + return nil, nil, err + } + + root := new(actionRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Event, resp, err +} + +func (s *DropletActionsServiceOp) doActionByTag(ctx context.Context, tag string, request *ActionRequest) ([]Action, *Response, error) { + if tag == "" { + return nil, nil, NewArgError("tag", "cannot be empty") + } + + if request == nil { + return nil, nil, NewArgError("request", "request can't be nil") + } + + path := dropletActionPathByTag(tag) + + req, err := s.client.NewRequest(ctx, http.MethodPost, path, request) + if err != nil { + return nil, nil, err + } + + root := new(actionsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Actions, resp, err +} + +// Get an action for a particular Droplet by id. +func (s *DropletActionsServiceOp) Get(ctx context.Context, dropletID, actionID int) (*Action, *Response, error) { + if dropletID < 1 { + return nil, nil, NewArgError("dropletID", "cannot be less than 1") + } + + if actionID < 1 { + return nil, nil, NewArgError("actionID", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d", dropletActionPath(dropletID), actionID) + return s.get(ctx, path) +} + +// GetByURI gets an action for a particular Droplet by id. +func (s *DropletActionsServiceOp) GetByURI(ctx context.Context, rawurl string) (*Action, *Response, error) { + u, err := url.Parse(rawurl) + if err != nil { + return nil, nil, err + } + + return s.get(ctx, u.Path) + +} + +func (s *DropletActionsServiceOp) get(ctx context.Context, path string) (*Action, *Response, error) { + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(actionRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Event, resp, err + +} + +func dropletActionPath(dropletID int) string { + return fmt.Sprintf("v2/droplets/%d/actions", dropletID) +} + +func dropletActionPathByTag(tag string) string { + return fmt.Sprintf("v2/droplets/actions?tag_name=%s", tag) +} diff --git a/vendor/github.com/digitalocean/godo/droplets.go b/vendor/github.com/digitalocean/godo/droplets.go new file mode 100644 index 0000000000000..72edf2b4382df --- /dev/null +++ b/vendor/github.com/digitalocean/godo/droplets.go @@ -0,0 +1,592 @@ +package godo + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" +) + +const dropletBasePath = "v2/droplets" + +var errNoNetworks = errors.New("no networks have been defined") + +// DropletsService is an interface for interfacing with the Droplet +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2#droplets +type DropletsService interface { + List(context.Context, *ListOptions) ([]Droplet, *Response, error) + ListByTag(context.Context, string, *ListOptions) ([]Droplet, *Response, error) + Get(context.Context, int) (*Droplet, *Response, error) + Create(context.Context, *DropletCreateRequest) (*Droplet, *Response, error) + CreateMultiple(context.Context, *DropletMultiCreateRequest) ([]Droplet, *Response, error) + Delete(context.Context, int) (*Response, error) + DeleteByTag(context.Context, string) (*Response, error) + Kernels(context.Context, int, *ListOptions) ([]Kernel, *Response, error) + Snapshots(context.Context, int, *ListOptions) ([]Image, *Response, error) + Backups(context.Context, int, *ListOptions) ([]Image, *Response, error) + Actions(context.Context, int, *ListOptions) ([]Action, *Response, error) + Neighbors(context.Context, int) ([]Droplet, *Response, error) +} + +// DropletsServiceOp handles communication with the Droplet related methods of the +// DigitalOcean API. +type DropletsServiceOp struct { + client *Client +} + +var _ DropletsService = &DropletsServiceOp{} + +// Droplet represents a DigitalOcean Droplet +type Droplet struct { + ID int `json:"id,float64,omitempty"` + Name string `json:"name,omitempty"` + Memory int `json:"memory,omitempty"` + Vcpus int `json:"vcpus,omitempty"` + Disk int `json:"disk,omitempty"` + Region *Region `json:"region,omitempty"` + Image *Image `json:"image,omitempty"` + Size *Size `json:"size,omitempty"` + SizeSlug string `json:"size_slug,omitempty"` + BackupIDs []int `json:"backup_ids,omitempty"` + NextBackupWindow *BackupWindow `json:"next_backup_window,omitempty"` + SnapshotIDs []int `json:"snapshot_ids,omitempty"` + Features []string `json:"features,omitempty"` + Locked bool `json:"locked,bool,omitempty"` + Status string `json:"status,omitempty"` + Networks *Networks `json:"networks,omitempty"` + Created string `json:"created_at,omitempty"` + Kernel *Kernel `json:"kernel,omitempty"` + Tags []string `json:"tags,omitempty"` + VolumeIDs []string `json:"volume_ids"` + VPCUUID string `json:"vpc_uuid,omitempty"` +} + +// PublicIPv4 returns the public IPv4 address for the Droplet. +func (d *Droplet) PublicIPv4() (string, error) { + if d.Networks == nil { + return "", errNoNetworks + } + + for _, v4 := range d.Networks.V4 { + if v4.Type == "public" { + return v4.IPAddress, nil + } + } + + return "", nil +} + +// PrivateIPv4 returns the private IPv4 address for the Droplet. +func (d *Droplet) PrivateIPv4() (string, error) { + if d.Networks == nil { + return "", errNoNetworks + } + + for _, v4 := range d.Networks.V4 { + if v4.Type == "private" { + return v4.IPAddress, nil + } + } + + return "", nil +} + +// PublicIPv6 returns the public IPv6 address for the Droplet. +func (d *Droplet) PublicIPv6() (string, error) { + if d.Networks == nil { + return "", errNoNetworks + } + + for _, v6 := range d.Networks.V6 { + if v6.Type == "public" { + return v6.IPAddress, nil + } + } + + return "", nil +} + +// Kernel object +type Kernel struct { + ID int `json:"id,float64,omitempty"` + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` +} + +// BackupWindow object +type BackupWindow struct { + Start *Timestamp `json:"start,omitempty"` + End *Timestamp `json:"end,omitempty"` +} + +// Convert Droplet to a string +func (d Droplet) String() string { + return Stringify(d) +} + +func (d Droplet) URN() string { + return ToURN("Droplet", d.ID) +} + +// DropletRoot represents a Droplet root +type dropletRoot struct { + Droplet *Droplet `json:"droplet"` + Links *Links `json:"links,omitempty"` +} + +type dropletsRoot struct { + Droplets []Droplet `json:"droplets"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +type kernelsRoot struct { + Kernels []Kernel `json:"kernels,omitempty"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +type dropletSnapshotsRoot struct { + Snapshots []Image `json:"snapshots,omitempty"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +type backupsRoot struct { + Backups []Image `json:"backups,omitempty"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +// DropletCreateImage identifies an image for the create request. It prefers slug over ID. +type DropletCreateImage struct { + ID int + Slug string +} + +// MarshalJSON returns either the slug or id of the image. It returns the id +// if the slug is empty. +func (d DropletCreateImage) MarshalJSON() ([]byte, error) { + if d.Slug != "" { + return json.Marshal(d.Slug) + } + + return json.Marshal(d.ID) +} + +// DropletCreateVolume identifies a volume to attach for the create request. It +// prefers Name over ID, +type DropletCreateVolume struct { + ID string + Name string +} + +// MarshalJSON returns an object with either the name or id of the volume. It +// returns the id if the name is empty. +func (d DropletCreateVolume) MarshalJSON() ([]byte, error) { + if d.Name != "" { + return json.Marshal(struct { + Name string `json:"name"` + }{Name: d.Name}) + } + + return json.Marshal(struct { + ID string `json:"id"` + }{ID: d.ID}) +} + +// DropletCreateSSHKey identifies a SSH Key for the create request. It prefers fingerprint over ID. +type DropletCreateSSHKey struct { + ID int + Fingerprint string +} + +// MarshalJSON returns either the fingerprint or id of the ssh key. It returns +// the id if the fingerprint is empty. +func (d DropletCreateSSHKey) MarshalJSON() ([]byte, error) { + if d.Fingerprint != "" { + return json.Marshal(d.Fingerprint) + } + + return json.Marshal(d.ID) +} + +// DropletCreateRequest represents a request to create a Droplet. +type DropletCreateRequest struct { + Name string `json:"name"` + Region string `json:"region"` + Size string `json:"size"` + Image DropletCreateImage `json:"image"` + SSHKeys []DropletCreateSSHKey `json:"ssh_keys"` + Backups bool `json:"backups"` + IPv6 bool `json:"ipv6"` + PrivateNetworking bool `json:"private_networking"` + Monitoring bool `json:"monitoring"` + UserData string `json:"user_data,omitempty"` + Volumes []DropletCreateVolume `json:"volumes,omitempty"` + Tags []string `json:"tags"` + VPCUUID string `json:"vpc_uuid,omitempty"` +} + +// DropletMultiCreateRequest is a request to create multiple Droplets. +type DropletMultiCreateRequest struct { + Names []string `json:"names"` + Region string `json:"region"` + Size string `json:"size"` + Image DropletCreateImage `json:"image"` + SSHKeys []DropletCreateSSHKey `json:"ssh_keys"` + Backups bool `json:"backups"` + IPv6 bool `json:"ipv6"` + PrivateNetworking bool `json:"private_networking"` + Monitoring bool `json:"monitoring"` + UserData string `json:"user_data,omitempty"` + Tags []string `json:"tags"` + VPCUUID string `json:"vpc_uuid,omitempty"` +} + +func (d DropletCreateRequest) String() string { + return Stringify(d) +} + +func (d DropletMultiCreateRequest) String() string { + return Stringify(d) +} + +// Networks represents the Droplet's Networks. +type Networks struct { + V4 []NetworkV4 `json:"v4,omitempty"` + V6 []NetworkV6 `json:"v6,omitempty"` +} + +// NetworkV4 represents a DigitalOcean IPv4 Network. +type NetworkV4 struct { + IPAddress string `json:"ip_address,omitempty"` + Netmask string `json:"netmask,omitempty"` + Gateway string `json:"gateway,omitempty"` + Type string `json:"type,omitempty"` +} + +func (n NetworkV4) String() string { + return Stringify(n) +} + +// NetworkV6 represents a DigitalOcean IPv6 network. +type NetworkV6 struct { + IPAddress string `json:"ip_address,omitempty"` + Netmask int `json:"netmask,omitempty"` + Gateway string `json:"gateway,omitempty"` + Type string `json:"type,omitempty"` +} + +func (n NetworkV6) String() string { + return Stringify(n) +} + +// Performs a list request given a path. +func (s *DropletsServiceOp) list(ctx context.Context, path string) ([]Droplet, *Response, error) { + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(dropletsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Droplets, resp, err +} + +// List all Droplets. +func (s *DropletsServiceOp) List(ctx context.Context, opt *ListOptions) ([]Droplet, *Response, error) { + path := dropletBasePath + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + return s.list(ctx, path) +} + +// ListByTag lists all Droplets matched by a Tag. +func (s *DropletsServiceOp) ListByTag(ctx context.Context, tag string, opt *ListOptions) ([]Droplet, *Response, error) { + path := fmt.Sprintf("%s?tag_name=%s", dropletBasePath, tag) + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + return s.list(ctx, path) +} + +// Get individual Droplet. +func (s *DropletsServiceOp) Get(ctx context.Context, dropletID int) (*Droplet, *Response, error) { + if dropletID < 1 { + return nil, nil, NewArgError("dropletID", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d", dropletBasePath, dropletID) + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(dropletRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Droplet, resp, err +} + +// Create Droplet +func (s *DropletsServiceOp) Create(ctx context.Context, createRequest *DropletCreateRequest) (*Droplet, *Response, error) { + if createRequest == nil { + return nil, nil, NewArgError("createRequest", "cannot be nil") + } + + path := dropletBasePath + + req, err := s.client.NewRequest(ctx, http.MethodPost, path, createRequest) + if err != nil { + return nil, nil, err + } + + root := new(dropletRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + + return root.Droplet, resp, err +} + +// CreateMultiple creates multiple Droplets. +func (s *DropletsServiceOp) CreateMultiple(ctx context.Context, createRequest *DropletMultiCreateRequest) ([]Droplet, *Response, error) { + if createRequest == nil { + return nil, nil, NewArgError("createRequest", "cannot be nil") + } + + path := dropletBasePath + + req, err := s.client.NewRequest(ctx, http.MethodPost, path, createRequest) + if err != nil { + return nil, nil, err + } + + root := new(dropletsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + + return root.Droplets, resp, err +} + +// Performs a delete request given a path +func (s *DropletsServiceOp) delete(ctx context.Context, path string) (*Response, error) { + req, err := s.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + resp, err := s.client.Do(ctx, req, nil) + + return resp, err +} + +// Delete Droplet. +func (s *DropletsServiceOp) Delete(ctx context.Context, dropletID int) (*Response, error) { + if dropletID < 1 { + return nil, NewArgError("dropletID", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d", dropletBasePath, dropletID) + + return s.delete(ctx, path) +} + +// DeleteByTag deletes Droplets matched by a Tag. +func (s *DropletsServiceOp) DeleteByTag(ctx context.Context, tag string) (*Response, error) { + if tag == "" { + return nil, NewArgError("tag", "cannot be empty") + } + + path := fmt.Sprintf("%s?tag_name=%s", dropletBasePath, tag) + + return s.delete(ctx, path) +} + +// Kernels lists kernels available for a Droplet. +func (s *DropletsServiceOp) Kernels(ctx context.Context, dropletID int, opt *ListOptions) ([]Kernel, *Response, error) { + if dropletID < 1 { + return nil, nil, NewArgError("dropletID", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d/kernels", dropletBasePath, dropletID) + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(kernelsRoot) + resp, err := s.client.Do(ctx, req, root) + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Kernels, resp, err +} + +// Actions lists the actions for a Droplet. +func (s *DropletsServiceOp) Actions(ctx context.Context, dropletID int, opt *ListOptions) ([]Action, *Response, error) { + if dropletID < 1 { + return nil, nil, NewArgError("dropletID", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d/actions", dropletBasePath, dropletID) + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(actionsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Actions, resp, err +} + +// Backups lists the backups for a Droplet. +func (s *DropletsServiceOp) Backups(ctx context.Context, dropletID int, opt *ListOptions) ([]Image, *Response, error) { + if dropletID < 1 { + return nil, nil, NewArgError("dropletID", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d/backups", dropletBasePath, dropletID) + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(backupsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Backups, resp, err +} + +// Snapshots lists the snapshots available for a Droplet. +func (s *DropletsServiceOp) Snapshots(ctx context.Context, dropletID int, opt *ListOptions) ([]Image, *Response, error) { + if dropletID < 1 { + return nil, nil, NewArgError("dropletID", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d/snapshots", dropletBasePath, dropletID) + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(dropletSnapshotsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Snapshots, resp, err +} + +// Neighbors lists the neighbors for a Droplet. +func (s *DropletsServiceOp) Neighbors(ctx context.Context, dropletID int) ([]Droplet, *Response, error) { + if dropletID < 1 { + return nil, nil, NewArgError("dropletID", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d/neighbors", dropletBasePath, dropletID) + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(dropletsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Droplets, resp, err +} + +func (s *DropletsServiceOp) dropletActionStatus(ctx context.Context, uri string) (string, error) { + action, _, err := s.client.DropletActions.GetByURI(ctx, uri) + + if err != nil { + return "", err + } + + return action.Status, nil +} diff --git a/vendor/github.com/digitalocean/godo/errors.go b/vendor/github.com/digitalocean/godo/errors.go new file mode 100644 index 0000000000000..a65ebd76b6cea --- /dev/null +++ b/vendor/github.com/digitalocean/godo/errors.go @@ -0,0 +1,24 @@ +package godo + +import "fmt" + +// ArgError is an error that represents an error with an input to godo. It +// identifies the argument and the cause (if possible). +type ArgError struct { + arg string + reason string +} + +var _ error = &ArgError{} + +// NewArgError creates an InputError. +func NewArgError(arg, reason string) *ArgError { + return &ArgError{ + arg: arg, + reason: reason, + } +} + +func (e *ArgError) Error() string { + return fmt.Sprintf("%s is invalid because %s", e.arg, e.reason) +} diff --git a/vendor/github.com/digitalocean/godo/firewalls.go b/vendor/github.com/digitalocean/godo/firewalls.go new file mode 100644 index 0000000000000..8453e6645e056 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/firewalls.go @@ -0,0 +1,271 @@ +package godo + +import ( + "context" + "net/http" + "path" + "strconv" +) + +const firewallsBasePath = "/v2/firewalls" + +// FirewallsService is an interface for managing Firewalls with the DigitalOcean API. +// See: https://developers.digitalocean.com/documentation/v2/#firewalls +type FirewallsService interface { + Get(context.Context, string) (*Firewall, *Response, error) + Create(context.Context, *FirewallRequest) (*Firewall, *Response, error) + Update(context.Context, string, *FirewallRequest) (*Firewall, *Response, error) + Delete(context.Context, string) (*Response, error) + List(context.Context, *ListOptions) ([]Firewall, *Response, error) + ListByDroplet(context.Context, int, *ListOptions) ([]Firewall, *Response, error) + AddDroplets(context.Context, string, ...int) (*Response, error) + RemoveDroplets(context.Context, string, ...int) (*Response, error) + AddTags(context.Context, string, ...string) (*Response, error) + RemoveTags(context.Context, string, ...string) (*Response, error) + AddRules(context.Context, string, *FirewallRulesRequest) (*Response, error) + RemoveRules(context.Context, string, *FirewallRulesRequest) (*Response, error) +} + +// FirewallsServiceOp handles communication with Firewalls methods of the DigitalOcean API. +type FirewallsServiceOp struct { + client *Client +} + +// Firewall represents a DigitalOcean Firewall configuration. +type Firewall struct { + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + InboundRules []InboundRule `json:"inbound_rules"` + OutboundRules []OutboundRule `json:"outbound_rules"` + DropletIDs []int `json:"droplet_ids"` + Tags []string `json:"tags"` + Created string `json:"created_at"` + PendingChanges []PendingChange `json:"pending_changes"` +} + +// String creates a human-readable description of a Firewall. +func (fw Firewall) String() string { + return Stringify(fw) +} + +func (fw Firewall) URN() string { + return ToURN("Firewall", fw.ID) +} + +// FirewallRequest represents the configuration to be applied to an existing or a new Firewall. +type FirewallRequest struct { + Name string `json:"name"` + InboundRules []InboundRule `json:"inbound_rules"` + OutboundRules []OutboundRule `json:"outbound_rules"` + DropletIDs []int `json:"droplet_ids"` + Tags []string `json:"tags"` +} + +// FirewallRulesRequest represents rules configuration to be applied to an existing Firewall. +type FirewallRulesRequest struct { + InboundRules []InboundRule `json:"inbound_rules"` + OutboundRules []OutboundRule `json:"outbound_rules"` +} + +// InboundRule represents a DigitalOcean Firewall inbound rule. +type InboundRule struct { + Protocol string `json:"protocol,omitempty"` + PortRange string `json:"ports,omitempty"` + Sources *Sources `json:"sources"` +} + +// OutboundRule represents a DigitalOcean Firewall outbound rule. +type OutboundRule struct { + Protocol string `json:"protocol,omitempty"` + PortRange string `json:"ports,omitempty"` + Destinations *Destinations `json:"destinations"` +} + +// Sources represents a DigitalOcean Firewall InboundRule sources. +type Sources struct { + Addresses []string `json:"addresses,omitempty"` + Tags []string `json:"tags,omitempty"` + DropletIDs []int `json:"droplet_ids,omitempty"` + LoadBalancerUIDs []string `json:"load_balancer_uids,omitempty"` +} + +// PendingChange represents a DigitalOcean Firewall status details. +type PendingChange struct { + DropletID int `json:"droplet_id,omitempty"` + Removing bool `json:"removing,omitempty"` + Status string `json:"status,omitempty"` +} + +// Destinations represents a DigitalOcean Firewall OutboundRule destinations. +type Destinations struct { + Addresses []string `json:"addresses,omitempty"` + Tags []string `json:"tags,omitempty"` + DropletIDs []int `json:"droplet_ids,omitempty"` + LoadBalancerUIDs []string `json:"load_balancer_uids,omitempty"` +} + +var _ FirewallsService = &FirewallsServiceOp{} + +// Get an existing Firewall by its identifier. +func (fw *FirewallsServiceOp) Get(ctx context.Context, fID string) (*Firewall, *Response, error) { + path := path.Join(firewallsBasePath, fID) + + req, err := fw.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(firewallRoot) + resp, err := fw.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Firewall, resp, err +} + +// Create a new Firewall with a given configuration. +func (fw *FirewallsServiceOp) Create(ctx context.Context, fr *FirewallRequest) (*Firewall, *Response, error) { + req, err := fw.client.NewRequest(ctx, http.MethodPost, firewallsBasePath, fr) + if err != nil { + return nil, nil, err + } + + root := new(firewallRoot) + resp, err := fw.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Firewall, resp, err +} + +// Update an existing Firewall with new configuration. +func (fw *FirewallsServiceOp) Update(ctx context.Context, fID string, fr *FirewallRequest) (*Firewall, *Response, error) { + path := path.Join(firewallsBasePath, fID) + + req, err := fw.client.NewRequest(ctx, "PUT", path, fr) + if err != nil { + return nil, nil, err + } + + root := new(firewallRoot) + resp, err := fw.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Firewall, resp, err +} + +// Delete a Firewall by its identifier. +func (fw *FirewallsServiceOp) Delete(ctx context.Context, fID string) (*Response, error) { + path := path.Join(firewallsBasePath, fID) + return fw.createAndDoReq(ctx, http.MethodDelete, path, nil) +} + +// List Firewalls. +func (fw *FirewallsServiceOp) List(ctx context.Context, opt *ListOptions) ([]Firewall, *Response, error) { + path, err := addOptions(firewallsBasePath, opt) + if err != nil { + return nil, nil, err + } + + return fw.listHelper(ctx, path) +} + +// ListByDroplet Firewalls. +func (fw *FirewallsServiceOp) ListByDroplet(ctx context.Context, dID int, opt *ListOptions) ([]Firewall, *Response, error) { + basePath := path.Join(dropletBasePath, strconv.Itoa(dID), "firewalls") + path, err := addOptions(basePath, opt) + if err != nil { + return nil, nil, err + } + + return fw.listHelper(ctx, path) +} + +// AddDroplets to a Firewall. +func (fw *FirewallsServiceOp) AddDroplets(ctx context.Context, fID string, dropletIDs ...int) (*Response, error) { + path := path.Join(firewallsBasePath, fID, "droplets") + return fw.createAndDoReq(ctx, http.MethodPost, path, &dropletsRequest{IDs: dropletIDs}) +} + +// RemoveDroplets from a Firewall. +func (fw *FirewallsServiceOp) RemoveDroplets(ctx context.Context, fID string, dropletIDs ...int) (*Response, error) { + path := path.Join(firewallsBasePath, fID, "droplets") + return fw.createAndDoReq(ctx, http.MethodDelete, path, &dropletsRequest{IDs: dropletIDs}) +} + +// AddTags to a Firewall. +func (fw *FirewallsServiceOp) AddTags(ctx context.Context, fID string, tags ...string) (*Response, error) { + path := path.Join(firewallsBasePath, fID, "tags") + return fw.createAndDoReq(ctx, http.MethodPost, path, &tagsRequest{Tags: tags}) +} + +// RemoveTags from a Firewall. +func (fw *FirewallsServiceOp) RemoveTags(ctx context.Context, fID string, tags ...string) (*Response, error) { + path := path.Join(firewallsBasePath, fID, "tags") + return fw.createAndDoReq(ctx, http.MethodDelete, path, &tagsRequest{Tags: tags}) +} + +// AddRules to a Firewall. +func (fw *FirewallsServiceOp) AddRules(ctx context.Context, fID string, rr *FirewallRulesRequest) (*Response, error) { + path := path.Join(firewallsBasePath, fID, "rules") + return fw.createAndDoReq(ctx, http.MethodPost, path, rr) +} + +// RemoveRules from a Firewall. +func (fw *FirewallsServiceOp) RemoveRules(ctx context.Context, fID string, rr *FirewallRulesRequest) (*Response, error) { + path := path.Join(firewallsBasePath, fID, "rules") + return fw.createAndDoReq(ctx, http.MethodDelete, path, rr) +} + +type dropletsRequest struct { + IDs []int `json:"droplet_ids"` +} + +type tagsRequest struct { + Tags []string `json:"tags"` +} + +type firewallRoot struct { + Firewall *Firewall `json:"firewall"` +} + +type firewallsRoot struct { + Firewalls []Firewall `json:"firewalls"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +func (fw *FirewallsServiceOp) createAndDoReq(ctx context.Context, method, path string, v interface{}) (*Response, error) { + req, err := fw.client.NewRequest(ctx, method, path, v) + if err != nil { + return nil, err + } + + return fw.client.Do(ctx, req, nil) +} + +func (fw *FirewallsServiceOp) listHelper(ctx context.Context, path string) ([]Firewall, *Response, error) { + req, err := fw.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(firewallsRoot) + resp, err := fw.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Firewalls, resp, err +} diff --git a/vendor/github.com/digitalocean/godo/floating_ips.go b/vendor/github.com/digitalocean/godo/floating_ips.go new file mode 100644 index 0000000000000..1720d767b4a42 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/floating_ips.go @@ -0,0 +1,143 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +const floatingBasePath = "v2/floating_ips" + +// FloatingIPsService is an interface for interfacing with the floating IPs +// endpoints of the Digital Ocean API. +// See: https://developers.digitalocean.com/documentation/v2#floating-ips +type FloatingIPsService interface { + List(context.Context, *ListOptions) ([]FloatingIP, *Response, error) + Get(context.Context, string) (*FloatingIP, *Response, error) + Create(context.Context, *FloatingIPCreateRequest) (*FloatingIP, *Response, error) + Delete(context.Context, string) (*Response, error) +} + +// FloatingIPsServiceOp handles communication with the floating IPs related methods of the +// DigitalOcean API. +type FloatingIPsServiceOp struct { + client *Client +} + +var _ FloatingIPsService = &FloatingIPsServiceOp{} + +// FloatingIP represents a Digital Ocean floating IP. +type FloatingIP struct { + Region *Region `json:"region"` + Droplet *Droplet `json:"droplet"` + IP string `json:"ip"` +} + +func (f FloatingIP) String() string { + return Stringify(f) +} + +func (f FloatingIP) URN() string { + return ToURN("FloatingIP", f.IP) +} + +type floatingIPsRoot struct { + FloatingIPs []FloatingIP `json:"floating_ips"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +type floatingIPRoot struct { + FloatingIP *FloatingIP `json:"floating_ip"` + Links *Links `json:"links,omitempty"` +} + +// FloatingIPCreateRequest represents a request to create a floating IP. +// If DropletID is not empty, the floating IP will be assigned to the +// droplet. +type FloatingIPCreateRequest struct { + Region string `json:"region"` + DropletID int `json:"droplet_id,omitempty"` +} + +// List all floating IPs. +func (f *FloatingIPsServiceOp) List(ctx context.Context, opt *ListOptions) ([]FloatingIP, *Response, error) { + path := floatingBasePath + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := f.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(floatingIPsRoot) + resp, err := f.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.FloatingIPs, resp, err +} + +// Get an individual floating IP. +func (f *FloatingIPsServiceOp) Get(ctx context.Context, ip string) (*FloatingIP, *Response, error) { + path := fmt.Sprintf("%s/%s", floatingBasePath, ip) + + req, err := f.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(floatingIPRoot) + resp, err := f.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.FloatingIP, resp, err +} + +// Create a floating IP. If the DropletID field of the request is not empty, +// the floating IP will also be assigned to the droplet. +func (f *FloatingIPsServiceOp) Create(ctx context.Context, createRequest *FloatingIPCreateRequest) (*FloatingIP, *Response, error) { + path := floatingBasePath + + req, err := f.client.NewRequest(ctx, http.MethodPost, path, createRequest) + if err != nil { + return nil, nil, err + } + + root := new(floatingIPRoot) + resp, err := f.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + + return root.FloatingIP, resp, err +} + +// Delete a floating IP. +func (f *FloatingIPsServiceOp) Delete(ctx context.Context, ip string) (*Response, error) { + path := fmt.Sprintf("%s/%s", floatingBasePath, ip) + + req, err := f.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + resp, err := f.client.Do(ctx, req, nil) + + return resp, err +} diff --git a/vendor/github.com/digitalocean/godo/floating_ips_actions.go b/vendor/github.com/digitalocean/godo/floating_ips_actions.go new file mode 100644 index 0000000000000..74ad279f95000 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/floating_ips_actions.go @@ -0,0 +1,109 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +// FloatingIPActionsService is an interface for interfacing with the +// floating IPs actions endpoints of the Digital Ocean API. +// See: https://developers.digitalocean.com/documentation/v2#floating-ips-action +type FloatingIPActionsService interface { + Assign(ctx context.Context, ip string, dropletID int) (*Action, *Response, error) + Unassign(ctx context.Context, ip string) (*Action, *Response, error) + Get(ctx context.Context, ip string, actionID int) (*Action, *Response, error) + List(ctx context.Context, ip string, opt *ListOptions) ([]Action, *Response, error) +} + +// FloatingIPActionsServiceOp handles communication with the floating IPs +// action related methods of the DigitalOcean API. +type FloatingIPActionsServiceOp struct { + client *Client +} + +// Assign a floating IP to a droplet. +func (s *FloatingIPActionsServiceOp) Assign(ctx context.Context, ip string, dropletID int) (*Action, *Response, error) { + request := &ActionRequest{ + "type": "assign", + "droplet_id": dropletID, + } + return s.doAction(ctx, ip, request) +} + +// Unassign a floating IP from the droplet it is currently assigned to. +func (s *FloatingIPActionsServiceOp) Unassign(ctx context.Context, ip string) (*Action, *Response, error) { + request := &ActionRequest{"type": "unassign"} + return s.doAction(ctx, ip, request) +} + +// Get an action for a particular floating IP by id. +func (s *FloatingIPActionsServiceOp) Get(ctx context.Context, ip string, actionID int) (*Action, *Response, error) { + path := fmt.Sprintf("%s/%d", floatingIPActionPath(ip), actionID) + return s.get(ctx, path) +} + +// List the actions for a particular floating IP. +func (s *FloatingIPActionsServiceOp) List(ctx context.Context, ip string, opt *ListOptions) ([]Action, *Response, error) { + path := floatingIPActionPath(ip) + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + return s.list(ctx, path) +} + +func (s *FloatingIPActionsServiceOp) doAction(ctx context.Context, ip string, request *ActionRequest) (*Action, *Response, error) { + path := floatingIPActionPath(ip) + + req, err := s.client.NewRequest(ctx, http.MethodPost, path, request) + if err != nil { + return nil, nil, err + } + + root := new(actionRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Event, resp, err +} + +func (s *FloatingIPActionsServiceOp) get(ctx context.Context, path string) (*Action, *Response, error) { + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(actionRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Event, resp, err +} + +func (s *FloatingIPActionsServiceOp) list(ctx context.Context, path string) ([]Action, *Response, error) { + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(actionsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + + return root.Actions, resp, err +} + +func floatingIPActionPath(ip string) string { + return fmt.Sprintf("%s/%s/actions", floatingBasePath, ip) +} diff --git a/vendor/github.com/digitalocean/godo/go.mod b/vendor/github.com/digitalocean/godo/go.mod new file mode 100644 index 0000000000000..36753b19f6cba --- /dev/null +++ b/vendor/github.com/digitalocean/godo/go.mod @@ -0,0 +1,16 @@ +module github.com/digitalocean/godo + +go 1.14 + +require ( + github.com/golang/protobuf v1.3.5 // indirect + github.com/google/go-querystring v1.0.0 + github.com/stretchr/testify v1.4.0 + golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect + golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d + google.golang.org/appengine v1.6.5 // indirect +) + +replace github.com/stretchr/objx => github.com/stretchr/objx v0.2.0 + +replace golang.org/x/crypto => golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a diff --git a/vendor/github.com/digitalocean/godo/go.sum b/vendor/github.com/digitalocean/godo/go.sum new file mode 100644 index 0000000000000..ccd0f08af3ce0 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/go.sum @@ -0,0 +1,42 @@ +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e h1:bRhVy7zSSasaqNksaRZiA5EEI+Ei4I1nO5Jh72wfHlg= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/vendor/github.com/digitalocean/godo/godo.go b/vendor/github.com/digitalocean/godo/godo.go new file mode 100644 index 0000000000000..c6dde8e21b3a6 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/godo.go @@ -0,0 +1,432 @@ +package godo + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "reflect" + "strconv" + "time" + + "github.com/google/go-querystring/query" + "golang.org/x/oauth2" +) + +const ( + libraryVersion = "1.37.0" + defaultBaseURL = "https://api.digitalocean.com/" + userAgent = "godo/" + libraryVersion + mediaType = "application/json" + + headerRateLimit = "RateLimit-Limit" + headerRateRemaining = "RateLimit-Remaining" + headerRateReset = "RateLimit-Reset" +) + +// Client manages communication with DigitalOcean V2 API. +type Client struct { + // HTTP client used to communicate with the DO API. + client *http.Client + + // Base URL for API requests. + BaseURL *url.URL + + // User agent for client + UserAgent string + + // Rate contains the current rate limit for the client as determined by the most recent + // API call. + Rate Rate + + // Services used for communicating with the API + Account AccountService + Actions ActionsService + Balance BalanceService + BillingHistory BillingHistoryService + CDNs CDNService + Domains DomainsService + Droplets DropletsService + DropletActions DropletActionsService + Images ImagesService + ImageActions ImageActionsService + Invoices InvoicesService + Keys KeysService + Regions RegionsService + Sizes SizesService + FloatingIPs FloatingIPsService + FloatingIPActions FloatingIPActionsService + Snapshots SnapshotsService + Storage StorageService + StorageActions StorageActionsService + Tags TagsService + LoadBalancers LoadBalancersService + Certificates CertificatesService + Firewalls FirewallsService + Projects ProjectsService + Kubernetes KubernetesService + Registry RegistryService + Databases DatabasesService + VPCs VPCsService + OneClick OneClickService + + // Optional function called after every successful request made to the DO APIs + onRequestCompleted RequestCompletionCallback +} + +// RequestCompletionCallback defines the type of the request callback function +type RequestCompletionCallback func(*http.Request, *http.Response) + +// ListOptions specifies the optional parameters to various List methods that +// support pagination. +type ListOptions struct { + // For paginated result sets, page of results to retrieve. + Page int `url:"page,omitempty"` + + // For paginated result sets, the number of results to include per page. + PerPage int `url:"per_page,omitempty"` +} + +// Response is a DigitalOcean response. This wraps the standard http.Response returned from DigitalOcean. +type Response struct { + *http.Response + + // Links that were returned with the response. These are parsed from + // request body and not the header. + Links *Links + + // Meta describes generic information about the response. + Meta *Meta + + // Monitoring URI + Monitor string + + Rate +} + +// An ErrorResponse reports the error caused by an API request +type ErrorResponse struct { + // HTTP response that caused this error + Response *http.Response + + // Error message + Message string `json:"message"` + + // RequestID returned from the API, useful to contact support. + RequestID string `json:"request_id"` +} + +// Rate contains the rate limit for the current client. +type Rate struct { + // The number of request per hour the client is currently limited to. + Limit int `json:"limit"` + + // The number of remaining requests the client can make this hour. + Remaining int `json:"remaining"` + + // The time at which the current rate limit will reset. + Reset Timestamp `json:"reset"` +} + +func addOptions(s string, opt interface{}) (string, error) { + v := reflect.ValueOf(opt) + + if v.Kind() == reflect.Ptr && v.IsNil() { + return s, nil + } + + origURL, err := url.Parse(s) + if err != nil { + return s, err + } + + origValues := origURL.Query() + + newValues, err := query.Values(opt) + if err != nil { + return s, err + } + + for k, v := range newValues { + origValues[k] = v + } + + origURL.RawQuery = origValues.Encode() + return origURL.String(), nil +} + +// NewFromToken returns a new DigitalOcean API client with the given API +// token. +func NewFromToken(token string) *Client { + ctx := context.Background() + + config := &oauth2.Config{} + ts := config.TokenSource(ctx, &oauth2.Token{AccessToken: token}) + + return NewClient(oauth2.NewClient(ctx, ts)) +} + +// NewClient returns a new DigitalOcean API client, using the given +// http.Client to perform all requests. +// +// Users who wish to pass their own http.Client should use this method. If +// you're in need of further customization, the godo.New method allows more +// options, such as setting a custom URL or a custom user agent string. +func NewClient(httpClient *http.Client) *Client { + if httpClient == nil { + httpClient = http.DefaultClient + } + + baseURL, _ := url.Parse(defaultBaseURL) + + c := &Client{client: httpClient, BaseURL: baseURL, UserAgent: userAgent} + c.Account = &AccountServiceOp{client: c} + c.Actions = &ActionsServiceOp{client: c} + c.Balance = &BalanceServiceOp{client: c} + c.BillingHistory = &BillingHistoryServiceOp{client: c} + c.CDNs = &CDNServiceOp{client: c} + c.Certificates = &CertificatesServiceOp{client: c} + c.Domains = &DomainsServiceOp{client: c} + c.Droplets = &DropletsServiceOp{client: c} + c.DropletActions = &DropletActionsServiceOp{client: c} + c.Firewalls = &FirewallsServiceOp{client: c} + c.FloatingIPs = &FloatingIPsServiceOp{client: c} + c.FloatingIPActions = &FloatingIPActionsServiceOp{client: c} + c.Images = &ImagesServiceOp{client: c} + c.ImageActions = &ImageActionsServiceOp{client: c} + c.Invoices = &InvoicesServiceOp{client: c} + c.Keys = &KeysServiceOp{client: c} + c.LoadBalancers = &LoadBalancersServiceOp{client: c} + c.Projects = &ProjectsServiceOp{client: c} + c.Regions = &RegionsServiceOp{client: c} + c.Sizes = &SizesServiceOp{client: c} + c.Snapshots = &SnapshotsServiceOp{client: c} + c.Storage = &StorageServiceOp{client: c} + c.StorageActions = &StorageActionsServiceOp{client: c} + c.Tags = &TagsServiceOp{client: c} + c.Kubernetes = &KubernetesServiceOp{client: c} + c.Registry = &RegistryServiceOp{client: c} + c.Databases = &DatabasesServiceOp{client: c} + c.VPCs = &VPCsServiceOp{client: c} + c.OneClick = &OneClickServiceOp{client: c} + + return c +} + +// ClientOpt are options for New. +type ClientOpt func(*Client) error + +// New returns a new DigitalOcean API client instance. +func New(httpClient *http.Client, opts ...ClientOpt) (*Client, error) { + c := NewClient(httpClient) + for _, opt := range opts { + if err := opt(c); err != nil { + return nil, err + } + } + + return c, nil +} + +// SetBaseURL is a client option for setting the base URL. +func SetBaseURL(bu string) ClientOpt { + return func(c *Client) error { + u, err := url.Parse(bu) + if err != nil { + return err + } + + c.BaseURL = u + return nil + } +} + +// SetUserAgent is a client option for setting the user agent. +func SetUserAgent(ua string) ClientOpt { + return func(c *Client) error { + c.UserAgent = fmt.Sprintf("%s %s", ua, c.UserAgent) + return nil + } +} + +// NewRequest creates an API request. A relative URL can be provided in urlStr, which will be resolved to the +// BaseURL of the Client. Relative URLS should always be specified without a preceding slash. If specified, the +// value pointed to by body is JSON encoded and included in as the request body. +func (c *Client) NewRequest(ctx context.Context, method, urlStr string, body interface{}) (*http.Request, error) { + u, err := c.BaseURL.Parse(urlStr) + if err != nil { + return nil, err + } + + buf := new(bytes.Buffer) + if body != nil { + err = json.NewEncoder(buf).Encode(body) + if err != nil { + return nil, err + } + } + + req, err := http.NewRequest(method, u.String(), buf) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", mediaType) + req.Header.Add("Accept", mediaType) + req.Header.Add("User-Agent", c.UserAgent) + return req, nil +} + +// OnRequestCompleted sets the DO API request completion callback +func (c *Client) OnRequestCompleted(rc RequestCompletionCallback) { + c.onRequestCompleted = rc +} + +// newResponse creates a new Response for the provided http.Response +func newResponse(r *http.Response) *Response { + response := Response{Response: r} + response.populateRate() + + return &response +} + +// populateRate parses the rate related headers and populates the response Rate. +func (r *Response) populateRate() { + if limit := r.Header.Get(headerRateLimit); limit != "" { + r.Rate.Limit, _ = strconv.Atoi(limit) + } + if remaining := r.Header.Get(headerRateRemaining); remaining != "" { + r.Rate.Remaining, _ = strconv.Atoi(remaining) + } + if reset := r.Header.Get(headerRateReset); reset != "" { + if v, _ := strconv.ParseInt(reset, 10, 64); v != 0 { + r.Rate.Reset = Timestamp{time.Unix(v, 0)} + } + } +} + +// Do sends an API request and returns the API response. The API response is JSON decoded and stored in the value +// pointed to by v, or returned as an error if an API error has occurred. If v implements the io.Writer interface, +// the raw response will be written to v, without attempting to decode it. +func (c *Client) Do(ctx context.Context, req *http.Request, v interface{}) (*Response, error) { + resp, err := DoRequestWithClient(ctx, c.client, req) + if err != nil { + return nil, err + } + if c.onRequestCompleted != nil { + c.onRequestCompleted(req, resp) + } + + defer func() { + if rerr := resp.Body.Close(); err == nil { + err = rerr + } + }() + + response := newResponse(resp) + c.Rate = response.Rate + + err = CheckResponse(resp) + if err != nil { + return response, err + } + + if v != nil { + if w, ok := v.(io.Writer); ok { + _, err = io.Copy(w, resp.Body) + if err != nil { + return nil, err + } + } else { + err = json.NewDecoder(resp.Body).Decode(v) + if err != nil { + return nil, err + } + } + } + + return response, err +} + +// DoRequest submits an HTTP request. +func DoRequest(ctx context.Context, req *http.Request) (*http.Response, error) { + return DoRequestWithClient(ctx, http.DefaultClient, req) +} + +// DoRequestWithClient submits an HTTP request using the specified client. +func DoRequestWithClient( + ctx context.Context, + client *http.Client, + req *http.Request) (*http.Response, error) { + req = req.WithContext(ctx) + return client.Do(req) +} + +func (r *ErrorResponse) Error() string { + if r.RequestID != "" { + return fmt.Sprintf("%v %v: %d (request %q) %v", + r.Response.Request.Method, r.Response.Request.URL, r.Response.StatusCode, r.RequestID, r.Message) + } + return fmt.Sprintf("%v %v: %d %v", + r.Response.Request.Method, r.Response.Request.URL, r.Response.StatusCode, r.Message) +} + +// CheckResponse checks the API response for errors, and returns them if present. A response is considered an +// error if it has a status code outside the 200 range. API error responses are expected to have either no response +// body, or a JSON response body that maps to ErrorResponse. Any other response body will be silently ignored. +func CheckResponse(r *http.Response) error { + if c := r.StatusCode; c >= 200 && c <= 299 { + return nil + } + + errorResponse := &ErrorResponse{Response: r} + data, err := ioutil.ReadAll(r.Body) + if err == nil && len(data) > 0 { + err := json.Unmarshal(data, errorResponse) + if err != nil { + errorResponse.Message = string(data) + } + } + + return errorResponse +} + +func (r Rate) String() string { + return Stringify(r) +} + +// String is a helper routine that allocates a new string value +// to store v and returns a pointer to it. +func String(v string) *string { + p := new(string) + *p = v + return p +} + +// Int is a helper routine that allocates a new int32 value +// to store v and returns a pointer to it, but unlike Int32 +// its argument value is an int. +func Int(v int) *int { + p := new(int) + *p = v + return p +} + +// Bool is a helper routine that allocates a new bool value +// to store v and returns a pointer to it. +func Bool(v bool) *bool { + p := new(bool) + *p = v + return p +} + +// StreamToString converts a reader to a string +func StreamToString(stream io.Reader) string { + buf := new(bytes.Buffer) + _, _ = buf.ReadFrom(stream) + return buf.String() +} diff --git a/vendor/github.com/digitalocean/godo/image_actions.go b/vendor/github.com/digitalocean/godo/image_actions.go new file mode 100644 index 0000000000000..976f7c6872438 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/image_actions.go @@ -0,0 +1,102 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +// ImageActionsService is an interface for interfacing with the image actions +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2#image-actions +type ImageActionsService interface { + Get(context.Context, int, int) (*Action, *Response, error) + Transfer(context.Context, int, *ActionRequest) (*Action, *Response, error) + Convert(context.Context, int) (*Action, *Response, error) +} + +// ImageActionsServiceOp handles communition with the image action related methods of the +// DigitalOcean API. +type ImageActionsServiceOp struct { + client *Client +} + +var _ ImageActionsService = &ImageActionsServiceOp{} + +// Transfer an image +func (i *ImageActionsServiceOp) Transfer(ctx context.Context, imageID int, transferRequest *ActionRequest) (*Action, *Response, error) { + if imageID < 1 { + return nil, nil, NewArgError("imageID", "cannot be less than 1") + } + + if transferRequest == nil { + return nil, nil, NewArgError("transferRequest", "cannot be nil") + } + + path := fmt.Sprintf("v2/images/%d/actions", imageID) + + req, err := i.client.NewRequest(ctx, http.MethodPost, path, transferRequest) + if err != nil { + return nil, nil, err + } + + root := new(actionRoot) + resp, err := i.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Event, resp, err +} + +// Convert an image to a snapshot +func (i *ImageActionsServiceOp) Convert(ctx context.Context, imageID int) (*Action, *Response, error) { + if imageID < 1 { + return nil, nil, NewArgError("imageID", "cannont be less than 1") + } + + path := fmt.Sprintf("v2/images/%d/actions", imageID) + + convertRequest := &ActionRequest{ + "type": "convert", + } + + req, err := i.client.NewRequest(ctx, http.MethodPost, path, convertRequest) + if err != nil { + return nil, nil, err + } + + root := new(actionRoot) + resp, err := i.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Event, resp, err +} + +// Get an action for a particular image by id. +func (i *ImageActionsServiceOp) Get(ctx context.Context, imageID, actionID int) (*Action, *Response, error) { + if imageID < 1 { + return nil, nil, NewArgError("imageID", "cannot be less than 1") + } + + if actionID < 1 { + return nil, nil, NewArgError("actionID", "cannot be less than 1") + } + + path := fmt.Sprintf("v2/images/%d/actions/%d", imageID, actionID) + + req, err := i.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(actionRoot) + resp, err := i.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Event, resp, err +} diff --git a/vendor/github.com/digitalocean/godo/images.go b/vendor/github.com/digitalocean/godo/images.go new file mode 100644 index 0000000000000..64e72e75ebc6c --- /dev/null +++ b/vendor/github.com/digitalocean/godo/images.go @@ -0,0 +1,245 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +const imageBasePath = "v2/images" + +// ImagesService is an interface for interfacing with the images +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2#images +type ImagesService interface { + List(context.Context, *ListOptions) ([]Image, *Response, error) + ListDistribution(ctx context.Context, opt *ListOptions) ([]Image, *Response, error) + ListApplication(ctx context.Context, opt *ListOptions) ([]Image, *Response, error) + ListUser(ctx context.Context, opt *ListOptions) ([]Image, *Response, error) + ListByTag(ctx context.Context, tag string, opt *ListOptions) ([]Image, *Response, error) + GetByID(context.Context, int) (*Image, *Response, error) + GetBySlug(context.Context, string) (*Image, *Response, error) + Create(context.Context, *CustomImageCreateRequest) (*Image, *Response, error) + Update(context.Context, int, *ImageUpdateRequest) (*Image, *Response, error) + Delete(context.Context, int) (*Response, error) +} + +// ImagesServiceOp handles communication with the image related methods of the +// DigitalOcean API. +type ImagesServiceOp struct { + client *Client +} + +var _ ImagesService = &ImagesServiceOp{} + +// Image represents a DigitalOcean Image +type Image struct { + ID int `json:"id,float64,omitempty"` + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + Distribution string `json:"distribution,omitempty"` + Slug string `json:"slug,omitempty"` + Public bool `json:"public,omitempty"` + Regions []string `json:"regions,omitempty"` + MinDiskSize int `json:"min_disk_size,omitempty"` + SizeGigaBytes float64 `json:"size_gigabytes,omitempty"` + Created string `json:"created_at,omitempty"` + Description string `json:"description,omitempty"` + Tags []string `json:"tags,omitempty"` + Status string `json:"status,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` +} + +// ImageUpdateRequest represents a request to update an image. +type ImageUpdateRequest struct { + Name string `json:"name"` +} + +// CustomImageCreateRequest represents a request to create a custom image. +type CustomImageCreateRequest struct { + Name string `json:"name"` + Url string `json:"url"` + Region string `json:"region"` + Distribution string `json:"distribution,omitempty"` + Description string `json:"description,omitempty"` + Tags []string `json:"tags,omitempty"` +} + +type imageRoot struct { + Image *Image +} + +type imagesRoot struct { + Images []Image + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +type listImageOptions struct { + Private bool `url:"private,omitempty"` + Type string `url:"type,omitempty"` + Tag string `url:"tag_name,omitempty"` +} + +func (i Image) String() string { + return Stringify(i) +} + +// List lists all the images available. +func (s *ImagesServiceOp) List(ctx context.Context, opt *ListOptions) ([]Image, *Response, error) { + return s.list(ctx, opt, nil) +} + +// ListDistribution lists all the distribution images. +func (s *ImagesServiceOp) ListDistribution(ctx context.Context, opt *ListOptions) ([]Image, *Response, error) { + listOpt := listImageOptions{Type: "distribution"} + return s.list(ctx, opt, &listOpt) +} + +// ListApplication lists all the application images. +func (s *ImagesServiceOp) ListApplication(ctx context.Context, opt *ListOptions) ([]Image, *Response, error) { + listOpt := listImageOptions{Type: "application"} + return s.list(ctx, opt, &listOpt) +} + +// ListUser lists all the user images. +func (s *ImagesServiceOp) ListUser(ctx context.Context, opt *ListOptions) ([]Image, *Response, error) { + listOpt := listImageOptions{Private: true} + return s.list(ctx, opt, &listOpt) +} + +// ListByTag lists all images with a specific tag applied. +func (s *ImagesServiceOp) ListByTag(ctx context.Context, tag string, opt *ListOptions) ([]Image, *Response, error) { + listOpt := listImageOptions{Tag: tag} + return s.list(ctx, opt, &listOpt) +} + +// GetByID retrieves an image by id. +func (s *ImagesServiceOp) GetByID(ctx context.Context, imageID int) (*Image, *Response, error) { + if imageID < 1 { + return nil, nil, NewArgError("imageID", "cannot be less than 1") + } + + return s.get(ctx, interface{}(imageID)) +} + +// GetBySlug retrieves an image by slug. +func (s *ImagesServiceOp) GetBySlug(ctx context.Context, slug string) (*Image, *Response, error) { + if len(slug) < 1 { + return nil, nil, NewArgError("slug", "cannot be blank") + } + + return s.get(ctx, interface{}(slug)) +} + +func (s *ImagesServiceOp) Create(ctx context.Context, createRequest *CustomImageCreateRequest) (*Image, *Response, error) { + if createRequest == nil { + return nil, nil, NewArgError("createRequest", "cannot be nil") + } + + req, err := s.client.NewRequest(ctx, http.MethodPost, imageBasePath, createRequest) + if err != nil { + return nil, nil, err + } + + root := new(imageRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Image, resp, err +} + +// Update an image name. +func (s *ImagesServiceOp) Update(ctx context.Context, imageID int, updateRequest *ImageUpdateRequest) (*Image, *Response, error) { + if imageID < 1 { + return nil, nil, NewArgError("imageID", "cannot be less than 1") + } + + if updateRequest == nil { + return nil, nil, NewArgError("updateRequest", "cannot be nil") + } + + path := fmt.Sprintf("%s/%d", imageBasePath, imageID) + req, err := s.client.NewRequest(ctx, http.MethodPut, path, updateRequest) + if err != nil { + return nil, nil, err + } + + root := new(imageRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Image, resp, err +} + +// Delete an image. +func (s *ImagesServiceOp) Delete(ctx context.Context, imageID int) (*Response, error) { + if imageID < 1 { + return nil, NewArgError("imageID", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d", imageBasePath, imageID) + + req, err := s.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + resp, err := s.client.Do(ctx, req, nil) + + return resp, err +} + +// Helper method for getting an individual image +func (s *ImagesServiceOp) get(ctx context.Context, ID interface{}) (*Image, *Response, error) { + path := fmt.Sprintf("%s/%v", imageBasePath, ID) + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(imageRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Image, resp, err +} + +// Helper method for listing images +func (s *ImagesServiceOp) list(ctx context.Context, opt *ListOptions, listOpt *listImageOptions) ([]Image, *Response, error) { + path := imageBasePath + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + path, err = addOptions(path, listOpt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(imagesRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Images, resp, err +} diff --git a/vendor/github.com/digitalocean/godo/invoices.go b/vendor/github.com/digitalocean/godo/invoices.go new file mode 100644 index 0000000000000..cc111f871ea81 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/invoices.go @@ -0,0 +1,225 @@ +package godo + +import ( + "bytes" + "context" + "fmt" + "net/http" + "time" +) + +const invoicesBasePath = "v2/customers/my/invoices" + +// InvoicesService is an interface for interfacing with the Invoice +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2/#invoices +type InvoicesService interface { + Get(context.Context, string, *ListOptions) (*Invoice, *Response, error) + GetPDF(context.Context, string) ([]byte, *Response, error) + GetCSV(context.Context, string) ([]byte, *Response, error) + List(context.Context, *ListOptions) (*InvoiceList, *Response, error) + GetSummary(context.Context, string) (*InvoiceSummary, *Response, error) +} + +// InvoicesServiceOp handles communication with the Invoice related methods of +// the DigitalOcean API. +type InvoicesServiceOp struct { + client *Client +} + +var _ InvoicesService = &InvoicesServiceOp{} + +// Invoice represents a DigitalOcean Invoice +type Invoice struct { + InvoiceItems []InvoiceItem `json:"invoice_items"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +// InvoiceItem represents a line-item on a DigitalOcean Invoice +type InvoiceItem struct { + Product string `json:"product"` + ResourceID string `json:"resource_id"` + ResourceUUID string `json:"resource_uuid"` + GroupDescription string `json:"group_description"` + Description string `json:"description"` + Amount string `json:"amount"` + Duration string `json:"duration"` + DurationUnit string `json:"duration_unit"` + StartTime time.Time `json:"start_time"` + EndTime time.Time `json:"end_time"` + ProjectName string `json:"project_name"` +} + +// InvoiceList contains a paginated list of all of a customer's invoices. +// The InvoicePreview is the month-to-date usage generated by DigitalOcean. +type InvoiceList struct { + Invoices []InvoiceListItem `json:"invoices"` + InvoicePreview InvoiceListItem `json:"invoice_preview"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +// InvoiceListItem contains a small list of information about a customer's invoice. +// More information can be found in the Invoice or InvoiceSummary +type InvoiceListItem struct { + InvoiceUUID string `json:"invoice_uuid"` + Amount string `json:"amount"` + InvoicePeriod string `json:"invoice_period"` + UpdatedAt time.Time `json:"updated_at"` +} + +// InvoiceSummary contains metadata and summarized usage for an invoice generated by DigitalOcean +type InvoiceSummary struct { + InvoiceUUID string `json:"invoice_uuid"` + BillingPeriod string `json:"billing_period"` + Amount string `json:"amount"` + UserName string `json:"user_name"` + UserBillingAddress Address `json:"user_billing_address"` + UserCompany string `json:"user_company"` + UserEmail string `json:"user_email"` + ProductCharges InvoiceSummaryBreakdown `json:"product_charges"` + Overages InvoiceSummaryBreakdown `json:"overages"` + Taxes InvoiceSummaryBreakdown `json:"taxes"` + CreditsAndAdjustments InvoiceSummaryBreakdown `json:"credits_and_adjustments"` +} + +// Address represents the billing address of a customer +type Address struct { + AddressLine1 string `json:"address_line1"` + AddressLine2 string `json:"address_line2"` + City string `json:"city"` + Region string `json:"region"` + PostalCode string `json:"postal_code"` + CountryISO2Code string `json:"country_iso2_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +// InvoiceSummaryBreakdown is a grouped set of InvoiceItems from an invoice +type InvoiceSummaryBreakdown struct { + Name string `json:"name"` + Amount string `json:"amount"` + Items []InvoiceSummaryBreakdownItem `json:"items"` +} + +// InvoiceSummaryBreakdownItem further breaks down the InvoiceSummary by product +type InvoiceSummaryBreakdownItem struct { + Name string `json:"name"` + Amount string `json:"amount"` + Count string `json:"count"` +} + +func (i Invoice) String() string { + return Stringify(i) +} + +// Get detailed invoice items for an Invoice +func (s *InvoicesServiceOp) Get(ctx context.Context, invoiceUUID string, opt *ListOptions) (*Invoice, *Response, error) { + path := fmt.Sprintf("%s/%s", invoicesBasePath, invoiceUUID) + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(Invoice) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root, resp, err +} + +// List invoices for a customer +func (s *InvoicesServiceOp) List(ctx context.Context, opt *ListOptions) (*InvoiceList, *Response, error) { + path := invoicesBasePath + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(InvoiceList) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root, resp, err +} + +// Get a summary of metadata and summarized usage for an Invoice +func (s *InvoicesServiceOp) GetSummary(ctx context.Context, invoiceUUID string) (*InvoiceSummary, *Response, error) { + path := fmt.Sprintf("%s/%s/summary", invoicesBasePath, invoiceUUID) + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(InvoiceSummary) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root, resp, err +} + +// Get the pdf for an Invoice +func (s *InvoicesServiceOp) GetPDF(ctx context.Context, invoiceUUID string) ([]byte, *Response, error) { + path := fmt.Sprintf("%s/%s/pdf", invoicesBasePath, invoiceUUID) + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + var root bytes.Buffer + resp, err := s.client.Do(ctx, req, &root) + if err != nil { + return nil, resp, err + } + + return root.Bytes(), resp, err +} + +// Get the csv for an Invoice +func (s *InvoicesServiceOp) GetCSV(ctx context.Context, invoiceUUID string) ([]byte, *Response, error) { + path := fmt.Sprintf("%s/%s/csv", invoicesBasePath, invoiceUUID) + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + var root bytes.Buffer + resp, err := s.client.Do(ctx, req, &root) + if err != nil { + return nil, resp, err + } + + return root.Bytes(), resp, err +} diff --git a/vendor/github.com/digitalocean/godo/keys.go b/vendor/github.com/digitalocean/godo/keys.go new file mode 100644 index 0000000000000..b97554d14cf0b --- /dev/null +++ b/vendor/github.com/digitalocean/godo/keys.go @@ -0,0 +1,230 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +const keysBasePath = "v2/account/keys" + +// KeysService is an interface for interfacing with the keys +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2#keys +type KeysService interface { + List(context.Context, *ListOptions) ([]Key, *Response, error) + GetByID(context.Context, int) (*Key, *Response, error) + GetByFingerprint(context.Context, string) (*Key, *Response, error) + Create(context.Context, *KeyCreateRequest) (*Key, *Response, error) + UpdateByID(context.Context, int, *KeyUpdateRequest) (*Key, *Response, error) + UpdateByFingerprint(context.Context, string, *KeyUpdateRequest) (*Key, *Response, error) + DeleteByID(context.Context, int) (*Response, error) + DeleteByFingerprint(context.Context, string) (*Response, error) +} + +// KeysServiceOp handles communication with key related method of the +// DigitalOcean API. +type KeysServiceOp struct { + client *Client +} + +var _ KeysService = &KeysServiceOp{} + +// Key represents a DigitalOcean Key. +type Key struct { + ID int `json:"id,float64,omitempty"` + Name string `json:"name,omitempty"` + Fingerprint string `json:"fingerprint,omitempty"` + PublicKey string `json:"public_key,omitempty"` +} + +// KeyUpdateRequest represents a request to update a DigitalOcean key. +type KeyUpdateRequest struct { + Name string `json:"name"` +} + +type keysRoot struct { + SSHKeys []Key `json:"ssh_keys"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +type keyRoot struct { + SSHKey *Key `json:"ssh_key"` +} + +func (s Key) String() string { + return Stringify(s) +} + +// KeyCreateRequest represents a request to create a new key. +type KeyCreateRequest struct { + Name string `json:"name"` + PublicKey string `json:"public_key"` +} + +// List all keys +func (s *KeysServiceOp) List(ctx context.Context, opt *ListOptions) ([]Key, *Response, error) { + path := keysBasePath + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(keysRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.SSHKeys, resp, err +} + +// Performs a get given a path +func (s *KeysServiceOp) get(ctx context.Context, path string) (*Key, *Response, error) { + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(keyRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.SSHKey, resp, err +} + +// GetByID gets a Key by id +func (s *KeysServiceOp) GetByID(ctx context.Context, keyID int) (*Key, *Response, error) { + if keyID < 1 { + return nil, nil, NewArgError("keyID", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d", keysBasePath, keyID) + return s.get(ctx, path) +} + +// GetByFingerprint gets a Key by by fingerprint +func (s *KeysServiceOp) GetByFingerprint(ctx context.Context, fingerprint string) (*Key, *Response, error) { + if len(fingerprint) < 1 { + return nil, nil, NewArgError("fingerprint", "cannot not be empty") + } + + path := fmt.Sprintf("%s/%s", keysBasePath, fingerprint) + return s.get(ctx, path) +} + +// Create a key using a KeyCreateRequest +func (s *KeysServiceOp) Create(ctx context.Context, createRequest *KeyCreateRequest) (*Key, *Response, error) { + if createRequest == nil { + return nil, nil, NewArgError("createRequest", "cannot be nil") + } + + req, err := s.client.NewRequest(ctx, http.MethodPost, keysBasePath, createRequest) + if err != nil { + return nil, nil, err + } + + root := new(keyRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.SSHKey, resp, err +} + +// UpdateByID updates a key name by ID. +func (s *KeysServiceOp) UpdateByID(ctx context.Context, keyID int, updateRequest *KeyUpdateRequest) (*Key, *Response, error) { + if keyID < 1 { + return nil, nil, NewArgError("keyID", "cannot be less than 1") + } + + if updateRequest == nil { + return nil, nil, NewArgError("updateRequest", "cannot be nil") + } + + path := fmt.Sprintf("%s/%d", keysBasePath, keyID) + req, err := s.client.NewRequest(ctx, "PUT", path, updateRequest) + if err != nil { + return nil, nil, err + } + + root := new(keyRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.SSHKey, resp, err +} + +// UpdateByFingerprint updates a key name by fingerprint. +func (s *KeysServiceOp) UpdateByFingerprint(ctx context.Context, fingerprint string, updateRequest *KeyUpdateRequest) (*Key, *Response, error) { + if len(fingerprint) < 1 { + return nil, nil, NewArgError("fingerprint", "cannot be empty") + } + + if updateRequest == nil { + return nil, nil, NewArgError("updateRequest", "cannot be nil") + } + + path := fmt.Sprintf("%s/%s", keysBasePath, fingerprint) + req, err := s.client.NewRequest(ctx, "PUT", path, updateRequest) + if err != nil { + return nil, nil, err + } + + root := new(keyRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.SSHKey, resp, err +} + +// Delete key using a path +func (s *KeysServiceOp) delete(ctx context.Context, path string) (*Response, error) { + req, err := s.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + resp, err := s.client.Do(ctx, req, nil) + + return resp, err +} + +// DeleteByID deletes a key by its id +func (s *KeysServiceOp) DeleteByID(ctx context.Context, keyID int) (*Response, error) { + if keyID < 1 { + return nil, NewArgError("keyID", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d", keysBasePath, keyID) + return s.delete(ctx, path) +} + +// DeleteByFingerprint deletes a key by its fingerprint +func (s *KeysServiceOp) DeleteByFingerprint(ctx context.Context, fingerprint string) (*Response, error) { + if len(fingerprint) < 1 { + return nil, NewArgError("fingerprint", "cannot be empty") + } + + path := fmt.Sprintf("%s/%s", keysBasePath, fingerprint) + return s.delete(ctx, path) +} diff --git a/vendor/github.com/digitalocean/godo/kubernetes.go b/vendor/github.com/digitalocean/godo/kubernetes.go new file mode 100644 index 0000000000000..9b80fefb6dc7b --- /dev/null +++ b/vendor/github.com/digitalocean/godo/kubernetes.go @@ -0,0 +1,697 @@ +package godo + +import ( + "bytes" + "context" + "encoding" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +const ( + kubernetesBasePath = "/v2/kubernetes" + kubernetesClustersPath = kubernetesBasePath + "/clusters" + kubernetesOptionsPath = kubernetesBasePath + "/options" +) + +// KubernetesService is an interface for interfacing with the Kubernetes endpoints +// of the DigitalOcean API. +// See: https://developers.digitalocean.com/documentation/v2#kubernetes +type KubernetesService interface { + Create(context.Context, *KubernetesClusterCreateRequest) (*KubernetesCluster, *Response, error) + Get(context.Context, string) (*KubernetesCluster, *Response, error) + GetUser(context.Context, string) (*KubernetesClusterUser, *Response, error) + GetUpgrades(context.Context, string) ([]*KubernetesVersion, *Response, error) + GetKubeConfig(context.Context, string) (*KubernetesClusterConfig, *Response, error) + GetCredentials(context.Context, string, *KubernetesClusterCredentialsGetRequest) (*KubernetesClusterCredentials, *Response, error) + List(context.Context, *ListOptions) ([]*KubernetesCluster, *Response, error) + Update(context.Context, string, *KubernetesClusterUpdateRequest) (*KubernetesCluster, *Response, error) + Upgrade(context.Context, string, *KubernetesClusterUpgradeRequest) (*Response, error) + Delete(context.Context, string) (*Response, error) + + CreateNodePool(ctx context.Context, clusterID string, req *KubernetesNodePoolCreateRequest) (*KubernetesNodePool, *Response, error) + GetNodePool(ctx context.Context, clusterID, poolID string) (*KubernetesNodePool, *Response, error) + ListNodePools(ctx context.Context, clusterID string, opts *ListOptions) ([]*KubernetesNodePool, *Response, error) + UpdateNodePool(ctx context.Context, clusterID, poolID string, req *KubernetesNodePoolUpdateRequest) (*KubernetesNodePool, *Response, error) + // RecycleNodePoolNodes is DEPRECATED please use DeleteNode + // The method will be removed in godo 2.0. + RecycleNodePoolNodes(ctx context.Context, clusterID, poolID string, req *KubernetesNodePoolRecycleNodesRequest) (*Response, error) + DeleteNodePool(ctx context.Context, clusterID, poolID string) (*Response, error) + DeleteNode(ctx context.Context, clusterID, poolID, nodeID string, req *KubernetesNodeDeleteRequest) (*Response, error) + + GetOptions(context.Context) (*KubernetesOptions, *Response, error) +} + +var _ KubernetesService = &KubernetesServiceOp{} + +// KubernetesServiceOp handles communication with Kubernetes methods of the DigitalOcean API. +type KubernetesServiceOp struct { + client *Client +} + +// KubernetesClusterCreateRequest represents a request to create a Kubernetes cluster. +type KubernetesClusterCreateRequest struct { + Name string `json:"name,omitempty"` + RegionSlug string `json:"region,omitempty"` + VersionSlug string `json:"version,omitempty"` + Tags []string `json:"tags,omitempty"` + VPCUUID string `json:"vpc_uuid,omitempty"` + + NodePools []*KubernetesNodePoolCreateRequest `json:"node_pools,omitempty"` + + MaintenancePolicy *KubernetesMaintenancePolicy `json:"maintenance_policy"` + AutoUpgrade bool `json:"auto_upgrade"` +} + +// KubernetesClusterUpdateRequest represents a request to update a Kubernetes cluster. +type KubernetesClusterUpdateRequest struct { + Name string `json:"name,omitempty"` + Tags []string `json:"tags,omitempty"` + MaintenancePolicy *KubernetesMaintenancePolicy `json:"maintenance_policy,omitempty"` + AutoUpgrade *bool `json:"auto_upgrade,omitempty"` +} + +// KubernetesClusterUpgradeRequest represents a request to upgrade a Kubernetes cluster. +type KubernetesClusterUpgradeRequest struct { + VersionSlug string `json:"version,omitempty"` +} + +// KubernetesNodePoolCreateRequest represents a request to create a node pool for a +// Kubernetes cluster. +type KubernetesNodePoolCreateRequest struct { + Name string `json:"name,omitempty"` + Size string `json:"size,omitempty"` + Count int `json:"count,omitempty"` + Tags []string `json:"tags,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + AutoScale bool `json:"auto_scale,omitempty"` + MinNodes int `json:"min_nodes,omitempty"` + MaxNodes int `json:"max_nodes,omitempty"` +} + +// KubernetesNodePoolUpdateRequest represents a request to update a node pool in a +// Kubernetes cluster. +type KubernetesNodePoolUpdateRequest struct { + Name string `json:"name,omitempty"` + Count *int `json:"count,omitempty"` + Tags []string `json:"tags,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + AutoScale *bool `json:"auto_scale,omitempty"` + MinNodes *int `json:"min_nodes,omitempty"` + MaxNodes *int `json:"max_nodes,omitempty"` +} + +// KubernetesNodePoolRecycleNodesRequest is DEPRECATED please use DeleteNode +// The type will be removed in godo 2.0. +type KubernetesNodePoolRecycleNodesRequest struct { + Nodes []string `json:"nodes,omitempty"` +} + +// KubernetesNodeDeleteRequest is a request to delete a specific node in a node pool. +type KubernetesNodeDeleteRequest struct { + // Replace will cause a new node to be created to replace the deleted node. + Replace bool `json:"replace,omitempty"` + + // SkipDrain skips draining the node before deleting it. + SkipDrain bool `json:"skip_drain,omitempty"` +} + +// KubernetesClusterCredentialsGetRequest is a request to get cluster credentials. +type KubernetesClusterCredentialsGetRequest struct { + ExpirySeconds *int `json:"expiry_seconds,omitempty"` +} + +// KubernetesCluster represents a Kubernetes cluster. +type KubernetesCluster struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + RegionSlug string `json:"region,omitempty"` + VersionSlug string `json:"version,omitempty"` + ClusterSubnet string `json:"cluster_subnet,omitempty"` + ServiceSubnet string `json:"service_subnet,omitempty"` + IPv4 string `json:"ipv4,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + Tags []string `json:"tags,omitempty"` + VPCUUID string `json:"vpc_uuid,omitempty"` + + NodePools []*KubernetesNodePool `json:"node_pools,omitempty"` + + MaintenancePolicy *KubernetesMaintenancePolicy `json:"maintenance_policy,omitempty"` + AutoUpgrade bool `json:"auto_upgrade,omitempty"` + + Status *KubernetesClusterStatus `json:"status,omitempty"` + CreatedAt time.Time `json:"created_at,omitempty"` + UpdatedAt time.Time `json:"updated_at,omitempty"` +} + +// KubernetesClusterUser represents a Kubernetes cluster user. +type KubernetesClusterUser struct { + Username string `json:"username,omitempty"` + Groups []string `json:"groups,omitempty"` +} + +// KubernetesClusterCredentials represents Kubernetes cluster credentials. +type KubernetesClusterCredentials struct { + Server string `json:"server"` + CertificateAuthorityData []byte `json:"certificate_authority_data"` + ClientCertificateData []byte `json:"client_certificate_data"` + ClientKeyData []byte `json:"client_key_data"` + Token string `json:"token"` + ExpiresAt time.Time `json:"expires_at"` +} + +// KubernetesMaintenancePolicy is a configuration to set the maintenance window +// of a cluster +type KubernetesMaintenancePolicy struct { + StartTime string `json:"start_time"` + Duration string `json:"duration"` + Day KubernetesMaintenancePolicyDay `json:"day"` +} + +// KubernetesMaintenancePolicyDay represents the possible days of a maintenance +// window +type KubernetesMaintenancePolicyDay int + +const ( + KubernetesMaintenanceDayAny KubernetesMaintenancePolicyDay = iota + KubernetesMaintenanceDayMonday + KubernetesMaintenanceDayTuesday + KubernetesMaintenanceDayWednesday + KubernetesMaintenanceDayThursday + KubernetesMaintenanceDayFriday + KubernetesMaintenanceDaySaturday + KubernetesMaintenanceDaySunday +) + +var ( + days = [...]string{ + "any", + "monday", + "tuesday", + "wednesday", + "thursday", + "friday", + "saturday", + "sunday", + } + + toDay = map[string]KubernetesMaintenancePolicyDay{ + "any": KubernetesMaintenanceDayAny, + "monday": KubernetesMaintenanceDayMonday, + "tuesday": KubernetesMaintenanceDayTuesday, + "wednesday": KubernetesMaintenanceDayWednesday, + "thursday": KubernetesMaintenanceDayThursday, + "friday": KubernetesMaintenanceDayFriday, + "saturday": KubernetesMaintenanceDaySaturday, + "sunday": KubernetesMaintenanceDaySunday, + } +) + +// KubernetesMaintenanceToDay returns the appropriate KubernetesMaintenancePolicyDay for the given string. +func KubernetesMaintenanceToDay(day string) (KubernetesMaintenancePolicyDay, error) { + d, ok := toDay[day] + if !ok { + return 0, fmt.Errorf("unknown day: %q", day) + } + + return d, nil +} + +func (k KubernetesMaintenancePolicyDay) String() string { + if KubernetesMaintenanceDayAny <= k && k <= KubernetesMaintenanceDaySunday { + return days[k] + } + return fmt.Sprintf("%d !Weekday", k) + +} + +func (k *KubernetesMaintenancePolicyDay) UnmarshalJSON(data []byte) error { + var val string + if err := json.Unmarshal(data, &val); err != nil { + return err + } + + parsed, err := KubernetesMaintenanceToDay(val) + if err != nil { + return err + } + *k = parsed + return nil +} + +func (k KubernetesMaintenancePolicyDay) MarshalJSON() ([]byte, error) { + if KubernetesMaintenanceDayAny <= k && k <= KubernetesMaintenanceDaySunday { + return json.Marshal(days[k]) + } + + return nil, fmt.Errorf("invalid day: %d", k) +} + +// Possible states for a cluster. +const ( + KubernetesClusterStatusProvisioning = KubernetesClusterStatusState("provisioning") + KubernetesClusterStatusRunning = KubernetesClusterStatusState("running") + KubernetesClusterStatusDegraded = KubernetesClusterStatusState("degraded") + KubernetesClusterStatusError = KubernetesClusterStatusState("error") + KubernetesClusterStatusDeleted = KubernetesClusterStatusState("deleted") + KubernetesClusterStatusUpgrading = KubernetesClusterStatusState("upgrading") + KubernetesClusterStatusInvalid = KubernetesClusterStatusState("invalid") +) + +// KubernetesClusterStatusState represents states for a cluster. +type KubernetesClusterStatusState string + +var _ encoding.TextUnmarshaler = (*KubernetesClusterStatusState)(nil) + +// UnmarshalText unmarshals the state. +func (s *KubernetesClusterStatusState) UnmarshalText(text []byte) error { + switch KubernetesClusterStatusState(strings.ToLower(string(text))) { + case KubernetesClusterStatusProvisioning: + *s = KubernetesClusterStatusProvisioning + case KubernetesClusterStatusRunning: + *s = KubernetesClusterStatusRunning + case KubernetesClusterStatusDegraded: + *s = KubernetesClusterStatusDegraded + case KubernetesClusterStatusError: + *s = KubernetesClusterStatusError + case KubernetesClusterStatusDeleted: + *s = KubernetesClusterStatusDeleted + case KubernetesClusterStatusUpgrading: + *s = KubernetesClusterStatusUpgrading + case "", KubernetesClusterStatusInvalid: + *s = KubernetesClusterStatusInvalid + default: + return fmt.Errorf("unknown cluster state %q", string(text)) + } + return nil +} + +// KubernetesClusterStatus describes the status of a cluster. +type KubernetesClusterStatus struct { + State KubernetesClusterStatusState `json:"state,omitempty"` + Message string `json:"message,omitempty"` +} + +// KubernetesNodePool represents a node pool in a Kubernetes cluster. +type KubernetesNodePool struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Size string `json:"size,omitempty"` + Count int `json:"count,omitempty"` + Tags []string `json:"tags,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + AutoScale bool `json:"auto_scale,omitempty"` + MinNodes int `json:"min_nodes,omitempty"` + MaxNodes int `json:"max_nodes,omitempty"` + + Nodes []*KubernetesNode `json:"nodes,omitempty"` +} + +// KubernetesNode represents a Node in a node pool in a Kubernetes cluster. +type KubernetesNode struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Status *KubernetesNodeStatus `json:"status,omitempty"` + DropletID string `json:"droplet_id,omitempty"` + + CreatedAt time.Time `json:"created_at,omitempty"` + UpdatedAt time.Time `json:"updated_at,omitempty"` +} + +// KubernetesNodeStatus represents the status of a particular Node in a Kubernetes cluster. +type KubernetesNodeStatus struct { + State string `json:"state,omitempty"` + Message string `json:"message,omitempty"` +} + +// KubernetesOptions represents options available for creating Kubernetes clusters. +type KubernetesOptions struct { + Versions []*KubernetesVersion `json:"versions,omitempty"` + Regions []*KubernetesRegion `json:"regions,omitempty"` + Sizes []*KubernetesNodeSize `json:"sizes,omitempty"` +} + +// KubernetesVersion is a DigitalOcean Kubernetes release. +type KubernetesVersion struct { + Slug string `json:"slug,omitempty"` + KubernetesVersion string `json:"kubernetes_version,omitempty"` +} + +// KubernetesNodeSize is a node sizes supported for Kubernetes clusters. +type KubernetesNodeSize struct { + Name string `json:"name"` + Slug string `json:"slug"` +} + +// KubernetesRegion is a region usable by Kubernetes clusters. +type KubernetesRegion struct { + Name string `json:"name"` + Slug string `json:"slug"` +} + +type kubernetesClustersRoot struct { + Clusters []*KubernetesCluster `json:"kubernetes_clusters,omitempty"` + Links *Links `json:"links,omitempty"` + Meta *Meta `json:"meta"` +} + +type kubernetesClusterRoot struct { + Cluster *KubernetesCluster `json:"kubernetes_cluster,omitempty"` +} + +type kubernetesClusterUserRoot struct { + User *KubernetesClusterUser `json:"kubernetes_cluster_user,omitempty"` +} + +type kubernetesNodePoolRoot struct { + NodePool *KubernetesNodePool `json:"node_pool,omitempty"` +} + +type kubernetesNodePoolsRoot struct { + NodePools []*KubernetesNodePool `json:"node_pools,omitempty"` + Links *Links `json:"links,omitempty"` +} + +type kubernetesUpgradesRoot struct { + AvailableUpgradeVersions []*KubernetesVersion `json:"available_upgrade_versions,omitempty"` +} + +// Get retrieves the details of a Kubernetes cluster. +func (svc *KubernetesServiceOp) Get(ctx context.Context, clusterID string) (*KubernetesCluster, *Response, error) { + path := fmt.Sprintf("%s/%s", kubernetesClustersPath, clusterID) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(kubernetesClusterRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Cluster, resp, nil +} + +// GetUser retrieves the details of a Kubernetes cluster user. +func (svc *KubernetesServiceOp) GetUser(ctx context.Context, clusterID string) (*KubernetesClusterUser, *Response, error) { + path := fmt.Sprintf("%s/%s/user", kubernetesClustersPath, clusterID) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(kubernetesClusterUserRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.User, resp, nil +} + +// GetUpgrades retrieves versions a Kubernetes cluster can be upgraded to. An +// upgrade can be requested using `Upgrade`. +func (svc *KubernetesServiceOp) GetUpgrades(ctx context.Context, clusterID string) ([]*KubernetesVersion, *Response, error) { + path := fmt.Sprintf("%s/%s/upgrades", kubernetesClustersPath, clusterID) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(kubernetesUpgradesRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, nil, err + } + return root.AvailableUpgradeVersions, resp, nil +} + +// Create creates a Kubernetes cluster. +func (svc *KubernetesServiceOp) Create(ctx context.Context, create *KubernetesClusterCreateRequest) (*KubernetesCluster, *Response, error) { + path := kubernetesClustersPath + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, create) + if err != nil { + return nil, nil, err + } + root := new(kubernetesClusterRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Cluster, resp, nil +} + +// Delete deletes a Kubernetes cluster. There is no way to recover a cluster +// once it has been destroyed. +func (svc *KubernetesServiceOp) Delete(ctx context.Context, clusterID string) (*Response, error) { + path := fmt.Sprintf("%s/%s", kubernetesClustersPath, clusterID) + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// List returns a list of the Kubernetes clusters visible with the caller's API token. +func (svc *KubernetesServiceOp) List(ctx context.Context, opts *ListOptions) ([]*KubernetesCluster, *Response, error) { + path := kubernetesClustersPath + path, err := addOptions(path, opts) + if err != nil { + return nil, nil, err + } + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(kubernetesClustersRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Clusters, resp, nil +} + +// KubernetesClusterConfig is the content of a Kubernetes config file, which can be +// used to interact with your Kubernetes cluster using `kubectl`. +// See: https://kubernetes.io/docs/tasks/tools/install-kubectl/ +type KubernetesClusterConfig struct { + KubeconfigYAML []byte +} + +// GetKubeConfig returns a Kubernetes config file for the specified cluster. +func (svc *KubernetesServiceOp) GetKubeConfig(ctx context.Context, clusterID string) (*KubernetesClusterConfig, *Response, error) { + path := fmt.Sprintf("%s/%s/kubeconfig", kubernetesClustersPath, clusterID) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + configBytes := bytes.NewBuffer(nil) + resp, err := svc.client.Do(ctx, req, configBytes) + if err != nil { + return nil, resp, err + } + res := &KubernetesClusterConfig{ + KubeconfigYAML: configBytes.Bytes(), + } + return res, resp, nil +} + +// GetCredentials returns a Kubernetes API server credentials for the specified cluster. +func (svc *KubernetesServiceOp) GetCredentials(ctx context.Context, clusterID string, get *KubernetesClusterCredentialsGetRequest) (*KubernetesClusterCredentials, *Response, error) { + path := fmt.Sprintf("%s/%s/credentials", kubernetesClustersPath, clusterID) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + q := req.URL.Query() + if get.ExpirySeconds != nil { + q.Add("expiry_seconds", strconv.Itoa(*get.ExpirySeconds)) + } + req.URL.RawQuery = q.Encode() + credentials := new(KubernetesClusterCredentials) + resp, err := svc.client.Do(ctx, req, credentials) + if err != nil { + return nil, nil, err + } + return credentials, resp, nil +} + +// Update updates a Kubernetes cluster's properties. +func (svc *KubernetesServiceOp) Update(ctx context.Context, clusterID string, update *KubernetesClusterUpdateRequest) (*KubernetesCluster, *Response, error) { + path := fmt.Sprintf("%s/%s", kubernetesClustersPath, clusterID) + req, err := svc.client.NewRequest(ctx, http.MethodPut, path, update) + if err != nil { + return nil, nil, err + } + root := new(kubernetesClusterRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Cluster, resp, nil +} + +// Upgrade upgrades a Kubernetes cluster to a new version. Valid upgrade +// versions for a given cluster can be retrieved with `GetUpgrades`. +func (svc *KubernetesServiceOp) Upgrade(ctx context.Context, clusterID string, upgrade *KubernetesClusterUpgradeRequest) (*Response, error) { + path := fmt.Sprintf("%s/%s/upgrade", kubernetesClustersPath, clusterID) + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, upgrade) + if err != nil { + return nil, err + } + return svc.client.Do(ctx, req, nil) +} + +// CreateNodePool creates a new node pool in an existing Kubernetes cluster. +func (svc *KubernetesServiceOp) CreateNodePool(ctx context.Context, clusterID string, create *KubernetesNodePoolCreateRequest) (*KubernetesNodePool, *Response, error) { + path := fmt.Sprintf("%s/%s/node_pools", kubernetesClustersPath, clusterID) + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, create) + if err != nil { + return nil, nil, err + } + root := new(kubernetesNodePoolRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.NodePool, resp, nil +} + +// GetNodePool retrieves an existing node pool in a Kubernetes cluster. +func (svc *KubernetesServiceOp) GetNodePool(ctx context.Context, clusterID, poolID string) (*KubernetesNodePool, *Response, error) { + path := fmt.Sprintf("%s/%s/node_pools/%s", kubernetesClustersPath, clusterID, poolID) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(kubernetesNodePoolRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.NodePool, resp, nil +} + +// ListNodePools lists all the node pools found in a Kubernetes cluster. +func (svc *KubernetesServiceOp) ListNodePools(ctx context.Context, clusterID string, opts *ListOptions) ([]*KubernetesNodePool, *Response, error) { + path := fmt.Sprintf("%s/%s/node_pools", kubernetesClustersPath, clusterID) + path, err := addOptions(path, opts) + if err != nil { + return nil, nil, err + } + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(kubernetesNodePoolsRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.NodePools, resp, nil +} + +// UpdateNodePool updates the details of an existing node pool. +func (svc *KubernetesServiceOp) UpdateNodePool(ctx context.Context, clusterID, poolID string, update *KubernetesNodePoolUpdateRequest) (*KubernetesNodePool, *Response, error) { + path := fmt.Sprintf("%s/%s/node_pools/%s", kubernetesClustersPath, clusterID, poolID) + req, err := svc.client.NewRequest(ctx, http.MethodPut, path, update) + if err != nil { + return nil, nil, err + } + root := new(kubernetesNodePoolRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.NodePool, resp, nil +} + +// RecycleNodePoolNodes is DEPRECATED please use DeleteNode +// The method will be removed in godo 2.0. +func (svc *KubernetesServiceOp) RecycleNodePoolNodes(ctx context.Context, clusterID, poolID string, recycle *KubernetesNodePoolRecycleNodesRequest) (*Response, error) { + path := fmt.Sprintf("%s/%s/node_pools/%s/recycle", kubernetesClustersPath, clusterID, poolID) + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, recycle) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// DeleteNodePool deletes a node pool, and subsequently all the nodes in that pool. +func (svc *KubernetesServiceOp) DeleteNodePool(ctx context.Context, clusterID, poolID string) (*Response, error) { + path := fmt.Sprintf("%s/%s/node_pools/%s", kubernetesClustersPath, clusterID, poolID) + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// DeleteNode deletes a specific node in a node pool. +func (svc *KubernetesServiceOp) DeleteNode(ctx context.Context, clusterID, poolID, nodeID string, deleteReq *KubernetesNodeDeleteRequest) (*Response, error) { + path := fmt.Sprintf("%s/%s/node_pools/%s/nodes/%s", kubernetesClustersPath, clusterID, poolID, nodeID) + if deleteReq != nil { + v := make(url.Values) + if deleteReq.SkipDrain { + v.Set("skip_drain", "1") + } + if deleteReq.Replace { + v.Set("replace", "1") + } + if query := v.Encode(); query != "" { + path = path + "?" + query + } + } + + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +type kubernetesOptionsRoot struct { + Options *KubernetesOptions `json:"options,omitempty"` + Links *Links `json:"links,omitempty"` +} + +// GetOptions returns options about the Kubernetes service, such as the versions available for +// cluster creation. +func (svc *KubernetesServiceOp) GetOptions(ctx context.Context) (*KubernetesOptions, *Response, error) { + path := kubernetesOptionsPath + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(kubernetesOptionsRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Options, resp, nil +} diff --git a/vendor/github.com/digitalocean/godo/links.go b/vendor/github.com/digitalocean/godo/links.go new file mode 100644 index 0000000000000..6f350bf9eaf34 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/links.go @@ -0,0 +1,83 @@ +package godo + +import ( + "context" + "net/url" + "strconv" +) + +// Links manages links that are returned along with a List +type Links struct { + Pages *Pages `json:"pages,omitempty"` + Actions []LinkAction `json:"actions,omitempty"` +} + +// Pages are pages specified in Links +type Pages struct { + First string `json:"first,omitempty"` + Prev string `json:"prev,omitempty"` + Last string `json:"last,omitempty"` + Next string `json:"next,omitempty"` +} + +// LinkAction is a pointer to an action +type LinkAction struct { + ID int `json:"id,omitempty"` + Rel string `json:"rel,omitempty"` + HREF string `json:"href,omitempty"` +} + +// CurrentPage is current page of the list +func (l *Links) CurrentPage() (int, error) { + return l.Pages.current() +} + +func (p *Pages) current() (int, error) { + switch { + case p == nil: + return 1, nil + case p.Prev == "" && p.Next != "": + return 1, nil + case p.Prev != "": + prevPage, err := pageForURL(p.Prev) + if err != nil { + return 0, err + } + + return prevPage + 1, nil + } + + return 0, nil +} + +// IsLastPage returns true if the current page is the last +func (l *Links) IsLastPage() bool { + if l.Pages == nil { + return true + } + return l.Pages.isLast() +} + +func (p *Pages) isLast() bool { + return p.Next == "" +} + +func pageForURL(urlText string) (int, error) { + u, err := url.ParseRequestURI(urlText) + if err != nil { + return 0, err + } + + pageStr := u.Query().Get("page") + page, err := strconv.Atoi(pageStr) + if err != nil { + return 0, err + } + + return page, nil +} + +// Get a link action by id. +func (la *LinkAction) Get(ctx context.Context, client *Client) (*Action, *Response, error) { + return client.Actions.Get(ctx, la.ID) +} diff --git a/vendor/github.com/digitalocean/godo/load_balancers.go b/vendor/github.com/digitalocean/godo/load_balancers.go new file mode 100644 index 0000000000000..86e9776fe4995 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/load_balancers.go @@ -0,0 +1,324 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +const loadBalancersBasePath = "/v2/load_balancers" +const forwardingRulesPath = "forwarding_rules" + +const dropletsPath = "droplets" + +// LoadBalancersService is an interface for managing load balancers with the DigitalOcean API. +// See: https://developers.digitalocean.com/documentation/v2#load-balancers +type LoadBalancersService interface { + Get(context.Context, string) (*LoadBalancer, *Response, error) + List(context.Context, *ListOptions) ([]LoadBalancer, *Response, error) + Create(context.Context, *LoadBalancerRequest) (*LoadBalancer, *Response, error) + Update(ctx context.Context, lbID string, lbr *LoadBalancerRequest) (*LoadBalancer, *Response, error) + Delete(ctx context.Context, lbID string) (*Response, error) + AddDroplets(ctx context.Context, lbID string, dropletIDs ...int) (*Response, error) + RemoveDroplets(ctx context.Context, lbID string, dropletIDs ...int) (*Response, error) + AddForwardingRules(ctx context.Context, lbID string, rules ...ForwardingRule) (*Response, error) + RemoveForwardingRules(ctx context.Context, lbID string, rules ...ForwardingRule) (*Response, error) +} + +// LoadBalancer represents a DigitalOcean load balancer configuration. +// Tags can only be provided upon the creation of a Load Balancer. +type LoadBalancer struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + IP string `json:"ip,omitempty"` + Algorithm string `json:"algorithm,omitempty"` + Status string `json:"status,omitempty"` + Created string `json:"created_at,omitempty"` + ForwardingRules []ForwardingRule `json:"forwarding_rules,omitempty"` + HealthCheck *HealthCheck `json:"health_check,omitempty"` + StickySessions *StickySessions `json:"sticky_sessions,omitempty"` + Region *Region `json:"region,omitempty"` + DropletIDs []int `json:"droplet_ids,omitempty"` + Tag string `json:"tag,omitempty"` + Tags []string `json:"tags,omitempty"` + RedirectHttpToHttps bool `json:"redirect_http_to_https,omitempty"` + EnableProxyProtocol bool `json:"enable_proxy_protocol,omitempty"` + EnableBackendKeepalive bool `json:"enable_backend_keepalive,omitempty"` + VPCUUID string `json:"vpc_uuid,omitempty"` +} + +// String creates a human-readable description of a LoadBalancer. +func (l LoadBalancer) String() string { + return Stringify(l) +} + +func (l LoadBalancer) URN() string { + return ToURN("LoadBalancer", l.ID) +} + +// AsRequest creates a LoadBalancerRequest that can be submitted to Update with the current values of the LoadBalancer. +// Modifying the returned LoadBalancerRequest will not modify the original LoadBalancer. +func (l LoadBalancer) AsRequest() *LoadBalancerRequest { + r := LoadBalancerRequest{ + Name: l.Name, + Algorithm: l.Algorithm, + ForwardingRules: append([]ForwardingRule(nil), l.ForwardingRules...), + DropletIDs: append([]int(nil), l.DropletIDs...), + Tag: l.Tag, + RedirectHttpToHttps: l.RedirectHttpToHttps, + EnableProxyProtocol: l.EnableProxyProtocol, + EnableBackendKeepalive: l.EnableBackendKeepalive, + HealthCheck: l.HealthCheck, + VPCUUID: l.VPCUUID, + } + + if l.HealthCheck != nil { + r.HealthCheck = &HealthCheck{} + *r.HealthCheck = *l.HealthCheck + } + if l.StickySessions != nil { + r.StickySessions = &StickySessions{} + *r.StickySessions = *l.StickySessions + } + if l.Region != nil { + r.Region = l.Region.Slug + } + return &r +} + +// ForwardingRule represents load balancer forwarding rules. +type ForwardingRule struct { + EntryProtocol string `json:"entry_protocol,omitempty"` + EntryPort int `json:"entry_port,omitempty"` + TargetProtocol string `json:"target_protocol,omitempty"` + TargetPort int `json:"target_port,omitempty"` + CertificateID string `json:"certificate_id,omitempty"` + TlsPassthrough bool `json:"tls_passthrough,omitempty"` +} + +// String creates a human-readable description of a ForwardingRule. +func (f ForwardingRule) String() string { + return Stringify(f) +} + +// HealthCheck represents optional load balancer health check rules. +type HealthCheck struct { + Protocol string `json:"protocol,omitempty"` + Port int `json:"port,omitempty"` + Path string `json:"path,omitempty"` + CheckIntervalSeconds int `json:"check_interval_seconds,omitempty"` + ResponseTimeoutSeconds int `json:"response_timeout_seconds,omitempty"` + HealthyThreshold int `json:"healthy_threshold,omitempty"` + UnhealthyThreshold int `json:"unhealthy_threshold,omitempty"` +} + +// String creates a human-readable description of a HealthCheck. +func (h HealthCheck) String() string { + return Stringify(h) +} + +// StickySessions represents optional load balancer session affinity rules. +type StickySessions struct { + Type string `json:"type,omitempty"` + CookieName string `json:"cookie_name,omitempty"` + CookieTtlSeconds int `json:"cookie_ttl_seconds,omitempty"` +} + +// String creates a human-readable description of a StickySessions instance. +func (s StickySessions) String() string { + return Stringify(s) +} + +// LoadBalancerRequest represents the configuration to be applied to an existing or a new load balancer. +type LoadBalancerRequest struct { + Name string `json:"name,omitempty"` + Algorithm string `json:"algorithm,omitempty"` + Region string `json:"region,omitempty"` + ForwardingRules []ForwardingRule `json:"forwarding_rules,omitempty"` + HealthCheck *HealthCheck `json:"health_check,omitempty"` + StickySessions *StickySessions `json:"sticky_sessions,omitempty"` + DropletIDs []int `json:"droplet_ids,omitempty"` + Tag string `json:"tag,omitempty"` + Tags []string `json:"tags,omitempty"` + RedirectHttpToHttps bool `json:"redirect_http_to_https,omitempty"` + EnableProxyProtocol bool `json:"enable_proxy_protocol,omitempty"` + EnableBackendKeepalive bool `json:"enable_backend_keepalive,omitempty"` + VPCUUID string `json:"vpc_uuid,omitempty"` +} + +// String creates a human-readable description of a LoadBalancerRequest. +func (l LoadBalancerRequest) String() string { + return Stringify(l) +} + +type forwardingRulesRequest struct { + Rules []ForwardingRule `json:"forwarding_rules,omitempty"` +} + +func (l forwardingRulesRequest) String() string { + return Stringify(l) +} + +type dropletIDsRequest struct { + IDs []int `json:"droplet_ids,omitempty"` +} + +func (l dropletIDsRequest) String() string { + return Stringify(l) +} + +type loadBalancersRoot struct { + LoadBalancers []LoadBalancer `json:"load_balancers"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +type loadBalancerRoot struct { + LoadBalancer *LoadBalancer `json:"load_balancer"` +} + +// LoadBalancersServiceOp handles communication with load balancer-related methods of the DigitalOcean API. +type LoadBalancersServiceOp struct { + client *Client +} + +var _ LoadBalancersService = &LoadBalancersServiceOp{} + +// Get an existing load balancer by its identifier. +func (l *LoadBalancersServiceOp) Get(ctx context.Context, lbID string) (*LoadBalancer, *Response, error) { + path := fmt.Sprintf("%s/%s", loadBalancersBasePath, lbID) + + req, err := l.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(loadBalancerRoot) + resp, err := l.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.LoadBalancer, resp, err +} + +// List load balancers, with optional pagination. +func (l *LoadBalancersServiceOp) List(ctx context.Context, opt *ListOptions) ([]LoadBalancer, *Response, error) { + path, err := addOptions(loadBalancersBasePath, opt) + if err != nil { + return nil, nil, err + } + + req, err := l.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(loadBalancersRoot) + resp, err := l.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.LoadBalancers, resp, err +} + +// Create a new load balancer with a given configuration. +func (l *LoadBalancersServiceOp) Create(ctx context.Context, lbr *LoadBalancerRequest) (*LoadBalancer, *Response, error) { + req, err := l.client.NewRequest(ctx, http.MethodPost, loadBalancersBasePath, lbr) + if err != nil { + return nil, nil, err + } + + root := new(loadBalancerRoot) + resp, err := l.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.LoadBalancer, resp, err +} + +// Update an existing load balancer with new configuration. +func (l *LoadBalancersServiceOp) Update(ctx context.Context, lbID string, lbr *LoadBalancerRequest) (*LoadBalancer, *Response, error) { + path := fmt.Sprintf("%s/%s", loadBalancersBasePath, lbID) + + req, err := l.client.NewRequest(ctx, "PUT", path, lbr) + if err != nil { + return nil, nil, err + } + + root := new(loadBalancerRoot) + resp, err := l.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.LoadBalancer, resp, err +} + +// Delete a load balancer by its identifier. +func (l *LoadBalancersServiceOp) Delete(ctx context.Context, ldID string) (*Response, error) { + path := fmt.Sprintf("%s/%s", loadBalancersBasePath, ldID) + + req, err := l.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + return l.client.Do(ctx, req, nil) +} + +// AddDroplets adds droplets to a load balancer. +func (l *LoadBalancersServiceOp) AddDroplets(ctx context.Context, lbID string, dropletIDs ...int) (*Response, error) { + path := fmt.Sprintf("%s/%s/%s", loadBalancersBasePath, lbID, dropletsPath) + + req, err := l.client.NewRequest(ctx, http.MethodPost, path, &dropletIDsRequest{IDs: dropletIDs}) + if err != nil { + return nil, err + } + + return l.client.Do(ctx, req, nil) +} + +// RemoveDroplets removes droplets from a load balancer. +func (l *LoadBalancersServiceOp) RemoveDroplets(ctx context.Context, lbID string, dropletIDs ...int) (*Response, error) { + path := fmt.Sprintf("%s/%s/%s", loadBalancersBasePath, lbID, dropletsPath) + + req, err := l.client.NewRequest(ctx, http.MethodDelete, path, &dropletIDsRequest{IDs: dropletIDs}) + if err != nil { + return nil, err + } + + return l.client.Do(ctx, req, nil) +} + +// AddForwardingRules adds forwarding rules to a load balancer. +func (l *LoadBalancersServiceOp) AddForwardingRules(ctx context.Context, lbID string, rules ...ForwardingRule) (*Response, error) { + path := fmt.Sprintf("%s/%s/%s", loadBalancersBasePath, lbID, forwardingRulesPath) + + req, err := l.client.NewRequest(ctx, http.MethodPost, path, &forwardingRulesRequest{Rules: rules}) + if err != nil { + return nil, err + } + + return l.client.Do(ctx, req, nil) +} + +// RemoveForwardingRules removes forwarding rules from a load balancer. +func (l *LoadBalancersServiceOp) RemoveForwardingRules(ctx context.Context, lbID string, rules ...ForwardingRule) (*Response, error) { + path := fmt.Sprintf("%s/%s/%s", loadBalancersBasePath, lbID, forwardingRulesPath) + + req, err := l.client.NewRequest(ctx, http.MethodDelete, path, &forwardingRulesRequest{Rules: rules}) + if err != nil { + return nil, err + } + + return l.client.Do(ctx, req, nil) +} diff --git a/vendor/github.com/digitalocean/godo/meta.go b/vendor/github.com/digitalocean/godo/meta.go new file mode 100644 index 0000000000000..d0b701785049f --- /dev/null +++ b/vendor/github.com/digitalocean/godo/meta.go @@ -0,0 +1,6 @@ +package godo + +// Meta describes generic information about a response. +type Meta struct { + Total int `json:"total"` +} diff --git a/vendor/github.com/digitalocean/godo/projects.go b/vendor/github.com/digitalocean/godo/projects.go new file mode 100644 index 0000000000000..172c2c9e2e3fb --- /dev/null +++ b/vendor/github.com/digitalocean/godo/projects.go @@ -0,0 +1,310 @@ +package godo + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "path" +) + +const ( + // DefaultProject is the ID you should use if you are working with your + // default project. + DefaultProject = "default" + + projectsBasePath = "/v2/projects" +) + +// ProjectsService is an interface for creating and managing Projects with the DigitalOcean API. +// See: https://developers.digitalocean.com/documentation/v2/#projects +type ProjectsService interface { + List(context.Context, *ListOptions) ([]Project, *Response, error) + GetDefault(context.Context) (*Project, *Response, error) + Get(context.Context, string) (*Project, *Response, error) + Create(context.Context, *CreateProjectRequest) (*Project, *Response, error) + Update(context.Context, string, *UpdateProjectRequest) (*Project, *Response, error) + Delete(context.Context, string) (*Response, error) + + ListResources(context.Context, string, *ListOptions) ([]ProjectResource, *Response, error) + AssignResources(context.Context, string, ...interface{}) ([]ProjectResource, *Response, error) +} + +// ProjectsServiceOp handles communication with Projects methods of the DigitalOcean API. +type ProjectsServiceOp struct { + client *Client +} + +// Project represents a DigitalOcean Project configuration. +type Project struct { + ID string `json:"id"` + OwnerUUID string `json:"owner_uuid"` + OwnerID uint64 `json:"owner_id"` + Name string `json:"name"` + Description string `json:"description"` + Purpose string `json:"purpose"` + Environment string `json:"environment"` + IsDefault bool `json:"is_default"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +// String creates a human-readable description of a Project. +func (p Project) String() string { + return Stringify(p) +} + +// CreateProjectRequest represents the request to create a new project. +type CreateProjectRequest struct { + Name string `json:"name"` + Description string `json:"description"` + Purpose string `json:"purpose"` + Environment string `json:"environment"` +} + +// UpdateProjectRequest represents the request to update project information. +// This type expects certain attribute types, but is built this way to allow +// nil values as well. See `updateProjectRequest` for the "real" types. +type UpdateProjectRequest struct { + Name interface{} + Description interface{} + Purpose interface{} + Environment interface{} + IsDefault interface{} +} + +type updateProjectRequest struct { + Name *string `json:"name"` + Description *string `json:"description"` + Purpose *string `json:"purpose"` + Environment *string `json:"environment"` + IsDefault *bool `json:"is_default"` +} + +// MarshalJSON takes an UpdateRequest and converts it to the "typed" request +// which is sent to the projects API. This is a PATCH request, which allows +// partial attributes, so `null` values are OK. +func (upr *UpdateProjectRequest) MarshalJSON() ([]byte, error) { + d := &updateProjectRequest{} + if str, ok := upr.Name.(string); ok { + d.Name = &str + } + if str, ok := upr.Description.(string); ok { + d.Description = &str + } + if str, ok := upr.Purpose.(string); ok { + d.Purpose = &str + } + if str, ok := upr.Environment.(string); ok { + d.Environment = &str + } + if val, ok := upr.IsDefault.(bool); ok { + d.IsDefault = &val + } + + return json.Marshal(d) +} + +type assignResourcesRequest struct { + Resources []string `json:"resources"` +} + +// ProjectResource is the projects API's representation of a resource. +type ProjectResource struct { + URN string `json:"urn"` + AssignedAt string `json:"assigned_at"` + Links *ProjectResourceLinks `json:"links"` + Status string `json:"status,omitempty"` +} + +// ProjetResourceLinks specify the link for more information about the resource. +type ProjectResourceLinks struct { + Self string `json:"self"` +} + +type projectsRoot struct { + Projects []Project `json:"projects"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +type projectRoot struct { + Project *Project `json:"project"` +} + +type projectResourcesRoot struct { + Resources []ProjectResource `json:"resources"` + Links *Links `json:"links,omitempty"` + Meta *Meta `json:"meta"` +} + +var _ ProjectsService = &ProjectsServiceOp{} + +// List Projects. +func (p *ProjectsServiceOp) List(ctx context.Context, opts *ListOptions) ([]Project, *Response, error) { + path, err := addOptions(projectsBasePath, opts) + if err != nil { + return nil, nil, err + } + + req, err := p.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(projectsRoot) + resp, err := p.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Projects, resp, err +} + +// GetDefault project. +func (p *ProjectsServiceOp) GetDefault(ctx context.Context) (*Project, *Response, error) { + return p.getHelper(ctx, "default") +} + +// Get retrieves a single project by its ID. +func (p *ProjectsServiceOp) Get(ctx context.Context, projectID string) (*Project, *Response, error) { + return p.getHelper(ctx, projectID) +} + +// Create a new project. +func (p *ProjectsServiceOp) Create(ctx context.Context, cr *CreateProjectRequest) (*Project, *Response, error) { + req, err := p.client.NewRequest(ctx, http.MethodPost, projectsBasePath, cr) + if err != nil { + return nil, nil, err + } + + root := new(projectRoot) + resp, err := p.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Project, resp, err +} + +// Update an existing project. +func (p *ProjectsServiceOp) Update(ctx context.Context, projectID string, ur *UpdateProjectRequest) (*Project, *Response, error) { + path := path.Join(projectsBasePath, projectID) + req, err := p.client.NewRequest(ctx, http.MethodPatch, path, ur) + if err != nil { + return nil, nil, err + } + + root := new(projectRoot) + resp, err := p.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Project, resp, err +} + +// Delete an existing project. You cannot have any resources in a project +// before deleting it. See the API documentation for more details. +func (p *ProjectsServiceOp) Delete(ctx context.Context, projectID string) (*Response, error) { + path := path.Join(projectsBasePath, projectID) + req, err := p.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + return p.client.Do(ctx, req, nil) +} + +// ListResources lists all resources in a project. +func (p *ProjectsServiceOp) ListResources(ctx context.Context, projectID string, opts *ListOptions) ([]ProjectResource, *Response, error) { + basePath := path.Join(projectsBasePath, projectID, "resources") + path, err := addOptions(basePath, opts) + if err != nil { + return nil, nil, err + } + + req, err := p.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(projectResourcesRoot) + resp, err := p.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Resources, resp, err +} + +// AssignResources assigns one or more resources to a project. AssignResources +// accepts resources in two possible formats: + +// 1. The resource type, like `&Droplet{ID: 1}` or `&FloatingIP{IP: "1.2.3.4"}` +// 2. A valid DO URN as a string, like "do:droplet:1234" +// +// There is no unassign. To move a resource to another project, just assign +// it to that other project. +func (p *ProjectsServiceOp) AssignResources(ctx context.Context, projectID string, resources ...interface{}) ([]ProjectResource, *Response, error) { + path := path.Join(projectsBasePath, projectID, "resources") + + ar := &assignResourcesRequest{ + Resources: make([]string, len(resources)), + } + + for i, resource := range resources { + switch resource := resource.(type) { + case ResourceWithURN: + ar.Resources[i] = resource.URN() + case string: + ar.Resources[i] = resource + default: + return nil, nil, fmt.Errorf("%T must either be a string or have a valid URN method", resource) + } + } + req, err := p.client.NewRequest(ctx, http.MethodPost, path, ar) + if err != nil { + return nil, nil, err + } + + root := new(projectResourcesRoot) + resp, err := p.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + + return root.Resources, resp, err +} + +func (p *ProjectsServiceOp) getHelper(ctx context.Context, projectID string) (*Project, *Response, error) { + path := path.Join(projectsBasePath, projectID) + + req, err := p.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(projectRoot) + resp, err := p.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Project, resp, err +} diff --git a/vendor/github.com/digitalocean/godo/regions.go b/vendor/github.com/digitalocean/godo/regions.go new file mode 100644 index 0000000000000..b07175e8a9f69 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/regions.go @@ -0,0 +1,68 @@ +package godo + +import ( + "context" + "net/http" +) + +// RegionsService is an interface for interfacing with the regions +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2#regions +type RegionsService interface { + List(context.Context, *ListOptions) ([]Region, *Response, error) +} + +// RegionsServiceOp handles communication with the region related methods of the +// DigitalOcean API. +type RegionsServiceOp struct { + client *Client +} + +var _ RegionsService = &RegionsServiceOp{} + +// Region represents a DigitalOcean Region +type Region struct { + Slug string `json:"slug,omitempty"` + Name string `json:"name,omitempty"` + Sizes []string `json:"sizes,omitempty"` + Available bool `json:"available,omitempty"` + Features []string `json:"features,omitempty"` +} + +type regionsRoot struct { + Regions []Region + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +func (r Region) String() string { + return Stringify(r) +} + +// List all regions +func (s *RegionsServiceOp) List(ctx context.Context, opt *ListOptions) ([]Region, *Response, error) { + path := "v2/regions" + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(regionsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Regions, resp, err +} diff --git a/vendor/github.com/digitalocean/godo/registry.go b/vendor/github.com/digitalocean/godo/registry.go new file mode 100644 index 0000000000000..1b5c40bd3d8cf --- /dev/null +++ b/vendor/github.com/digitalocean/godo/registry.go @@ -0,0 +1,253 @@ +package godo + +import ( + "bytes" + "context" + "fmt" + "net/http" + "net/url" + "strconv" + "time" +) + +const ( + registryPath = "/v2/registry" + // RegistryServer is the hostname of the DigitalOcean registry service + RegistryServer = "registry.digitalocean.com" +) + +// RegistryService is an interface for interfacing with the Registry endpoints +// of the DigitalOcean API. +// See: https://developers.digitalocean.com/documentation/v2#registry +type RegistryService interface { + Create(context.Context, *RegistryCreateRequest) (*Registry, *Response, error) + Get(context.Context) (*Registry, *Response, error) + Delete(context.Context) (*Response, error) + DockerCredentials(context.Context, *RegistryDockerCredentialsRequest) (*DockerCredentials, *Response, error) + ListRepositories(context.Context, string, *ListOptions) ([]*Repository, *Response, error) + ListRepositoryTags(context.Context, string, string, *ListOptions) ([]*RepositoryTag, *Response, error) + DeleteTag(context.Context, string, string, string) (*Response, error) + DeleteManifest(context.Context, string, string, string) (*Response, error) +} + +var _ RegistryService = &RegistryServiceOp{} + +// RegistryServiceOp handles communication with Registry methods of the DigitalOcean API. +type RegistryServiceOp struct { + client *Client +} + +// RegistryCreateRequest represents a request to create a registry. +type RegistryCreateRequest struct { + Name string `json:"name,omitempty"` +} + +// RegistryDockerCredentialsRequest represents a request to retrieve docker +// credentials for a registry. +type RegistryDockerCredentialsRequest struct { + ReadWrite bool `json:"read_write"` + ExpirySeconds *int `json:"expiry_seconds,omitempty"` +} + +// Registry represents a registry. +type Registry struct { + Name string `json:"name,omitempty"` + CreatedAt time.Time `json:"created_at,omitempty"` +} + +// Repository represents a repository +type Repository struct { + RegistryName string `json:"registry_name,omitempty"` + Name string `json:"name,omitempty"` + LatestTag *RepositoryTag `json:"latest_tag,omitempty"` + TagCount uint64 `json:"tag_count,omitempty"` +} + +// RepositoryTag represents a repository tag +type RepositoryTag struct { + RegistryName string `json:"registry_name,omitempty"` + Repository string `json:"repository,omitempty"` + Tag string `json:"tag,omitempty"` + ManifestDigest string `json:"manifest_digest,omitempty"` + CompressedSizeBytes uint64 `json:"compressed_size_bytes,omitempty"` + SizeBytes uint64 `json:"size_bytes,omitempty"` + UpdatedAt time.Time `json:"updated_at,omitempty"` +} + +type registryRoot struct { + Registry *Registry `json:"registry,omitempty"` +} + +type repositoriesRoot struct { + Repositories []*Repository `json:"repositories,omitempty"` + Links *Links `json:"links,omitempty"` + Meta *Meta `json:"meta"` +} + +type repositoryTagsRoot struct { + Tags []*RepositoryTag `json:"tags,omitempty"` + Links *Links `json:"links,omitempty"` + Meta *Meta `json:"meta"` +} + +// Get retrieves the details of a Registry. +func (svc *RegistryServiceOp) Get(ctx context.Context) (*Registry, *Response, error) { + req, err := svc.client.NewRequest(ctx, http.MethodGet, registryPath, nil) + if err != nil { + return nil, nil, err + } + root := new(registryRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Registry, resp, nil +} + +// Create creates a registry. +func (svc *RegistryServiceOp) Create(ctx context.Context, create *RegistryCreateRequest) (*Registry, *Response, error) { + req, err := svc.client.NewRequest(ctx, http.MethodPost, registryPath, create) + if err != nil { + return nil, nil, err + } + root := new(registryRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Registry, resp, nil +} + +// Delete deletes a registry. There is no way to recover a registry once it has +// been destroyed. +func (svc *RegistryServiceOp) Delete(ctx context.Context) (*Response, error) { + req, err := svc.client.NewRequest(ctx, http.MethodDelete, registryPath, nil) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// DockerCredentials is the content of a Docker config file +// that is used by the docker CLI +// See: https://docs.docker.com/engine/reference/commandline/cli/#configjson-properties +type DockerCredentials struct { + DockerConfigJSON []byte +} + +// DockerCredentials retrieves a Docker config file containing the registry's credentials. +func (svc *RegistryServiceOp) DockerCredentials(ctx context.Context, request *RegistryDockerCredentialsRequest) (*DockerCredentials, *Response, error) { + path := fmt.Sprintf("%s/%s", registryPath, "docker-credentials") + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + q := req.URL.Query() + q.Add("read_write", strconv.FormatBool(request.ReadWrite)) + if request.ExpirySeconds != nil { + q.Add("expiry_seconds", strconv.Itoa(*request.ExpirySeconds)) + } + req.URL.RawQuery = q.Encode() + + var buf bytes.Buffer + resp, err := svc.client.Do(ctx, req, &buf) + if err != nil { + return nil, resp, err + } + + dc := &DockerCredentials{ + DockerConfigJSON: buf.Bytes(), + } + return dc, resp, nil +} + +// ListRepositories returns a list of the Repositories visible with the registry's credentials. +func (svc *RegistryServiceOp) ListRepositories(ctx context.Context, registry string, opts *ListOptions) ([]*Repository, *Response, error) { + path := fmt.Sprintf("%s/%s/repositories", registryPath, registry) + path, err := addOptions(path, opts) + if err != nil { + return nil, nil, err + } + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(repositoriesRoot) + + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Repositories, resp, nil +} + +// ListRepositoryTags returns a list of the RepositoryTags available within the given repository. +func (svc *RegistryServiceOp) ListRepositoryTags(ctx context.Context, registry, repository string, opts *ListOptions) ([]*RepositoryTag, *Response, error) { + path := fmt.Sprintf("%s/%s/repositories/%s/tags", registryPath, registry, url.PathEscape(repository)) + path, err := addOptions(path, opts) + if err != nil { + return nil, nil, err + } + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(repositoryTagsRoot) + + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Tags, resp, nil +} + +// DeleteTag deletes a tag within a given repository. +func (svc *RegistryServiceOp) DeleteTag(ctx context.Context, registry, repository, tag string) (*Response, error) { + path := fmt.Sprintf("%s/%s/repositories/%s/tags/%s", registryPath, registry, url.PathEscape(repository), tag) + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + + return resp, nil +} + +// DeleteManifest deletes a manifest by its digest within a given repository. +func (svc *RegistryServiceOp) DeleteManifest(ctx context.Context, registry, repository, digest string) (*Response, error) { + path := fmt.Sprintf("%s/%s/repositories/%s/digests/%s", registryPath, registry, url.PathEscape(repository), digest) + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + + return resp, nil +} diff --git a/vendor/github.com/digitalocean/godo/sizes.go b/vendor/github.com/digitalocean/godo/sizes.go new file mode 100644 index 0000000000000..d2b93ea7fb474 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/sizes.go @@ -0,0 +1,72 @@ +package godo + +import ( + "context" + "net/http" +) + +// SizesService is an interface for interfacing with the size +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2#sizes +type SizesService interface { + List(context.Context, *ListOptions) ([]Size, *Response, error) +} + +// SizesServiceOp handles communication with the size related methods of the +// DigitalOcean API. +type SizesServiceOp struct { + client *Client +} + +var _ SizesService = &SizesServiceOp{} + +// Size represents a DigitalOcean Size +type Size struct { + Slug string `json:"slug,omitempty"` + Memory int `json:"memory,omitempty"` + Vcpus int `json:"vcpus,omitempty"` + Disk int `json:"disk,omitempty"` + PriceMonthly float64 `json:"price_monthly,omitempty"` + PriceHourly float64 `json:"price_hourly,omitempty"` + Regions []string `json:"regions,omitempty"` + Available bool `json:"available,omitempty"` + Transfer float64 `json:"transfer,omitempty"` +} + +func (s Size) String() string { + return Stringify(s) +} + +type sizesRoot struct { + Sizes []Size + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +// List all images +func (s *SizesServiceOp) List(ctx context.Context, opt *ListOptions) ([]Size, *Response, error) { + path := "v2/sizes" + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(sizesRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Sizes, resp, err +} diff --git a/vendor/github.com/digitalocean/godo/snapshots.go b/vendor/github.com/digitalocean/godo/snapshots.go new file mode 100644 index 0000000000000..cf95ccc007605 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/snapshots.go @@ -0,0 +1,142 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +const snapshotBasePath = "v2/snapshots" + +// SnapshotsService is an interface for interfacing with the snapshots +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2#snapshots +type SnapshotsService interface { + List(context.Context, *ListOptions) ([]Snapshot, *Response, error) + ListVolume(context.Context, *ListOptions) ([]Snapshot, *Response, error) + ListDroplet(context.Context, *ListOptions) ([]Snapshot, *Response, error) + Get(context.Context, string) (*Snapshot, *Response, error) + Delete(context.Context, string) (*Response, error) +} + +// SnapshotsServiceOp handles communication with the snapshot related methods of the +// DigitalOcean API. +type SnapshotsServiceOp struct { + client *Client +} + +var _ SnapshotsService = &SnapshotsServiceOp{} + +// Snapshot represents a DigitalOcean Snapshot +type Snapshot struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + ResourceID string `json:"resource_id,omitempty"` + ResourceType string `json:"resource_type,omitempty"` + Regions []string `json:"regions,omitempty"` + MinDiskSize int `json:"min_disk_size,omitempty"` + SizeGigaBytes float64 `json:"size_gigabytes,omitempty"` + Created string `json:"created_at,omitempty"` + Tags []string `json:"tags,omitempty"` +} + +type snapshotRoot struct { + Snapshot *Snapshot `json:"snapshot"` +} + +type snapshotsRoot struct { + Snapshots []Snapshot `json:"snapshots"` + Links *Links `json:"links,omitempty"` + Meta *Meta `json:"meta,omitempty"` +} + +type listSnapshotOptions struct { + ResourceType string `url:"resource_type,omitempty"` +} + +func (s Snapshot) String() string { + return Stringify(s) +} + +// List lists all the snapshots available. +func (s *SnapshotsServiceOp) List(ctx context.Context, opt *ListOptions) ([]Snapshot, *Response, error) { + return s.list(ctx, opt, nil) +} + +// ListDroplet lists all the Droplet snapshots. +func (s *SnapshotsServiceOp) ListDroplet(ctx context.Context, opt *ListOptions) ([]Snapshot, *Response, error) { + listOpt := listSnapshotOptions{ResourceType: "droplet"} + return s.list(ctx, opt, &listOpt) +} + +// ListVolume lists all the volume snapshots. +func (s *SnapshotsServiceOp) ListVolume(ctx context.Context, opt *ListOptions) ([]Snapshot, *Response, error) { + listOpt := listSnapshotOptions{ResourceType: "volume"} + return s.list(ctx, opt, &listOpt) +} + +// Get retrieves an snapshot by id. +func (s *SnapshotsServiceOp) Get(ctx context.Context, snapshotID string) (*Snapshot, *Response, error) { + return s.get(ctx, snapshotID) +} + +// Delete an snapshot. +func (s *SnapshotsServiceOp) Delete(ctx context.Context, snapshotID string) (*Response, error) { + path := fmt.Sprintf("%s/%s", snapshotBasePath, snapshotID) + + req, err := s.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + resp, err := s.client.Do(ctx, req, nil) + + return resp, err +} + +// Helper method for getting an individual snapshot +func (s *SnapshotsServiceOp) get(ctx context.Context, ID string) (*Snapshot, *Response, error) { + path := fmt.Sprintf("%s/%s", snapshotBasePath, ID) + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(snapshotRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Snapshot, resp, err +} + +// Helper method for listing snapshots +func (s *SnapshotsServiceOp) list(ctx context.Context, opt *ListOptions, listOpt *listSnapshotOptions) ([]Snapshot, *Response, error) { + path := snapshotBasePath + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + path, err = addOptions(path, listOpt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(snapshotsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + + return root.Snapshots, resp, err +} diff --git a/vendor/github.com/digitalocean/godo/storage.go b/vendor/github.com/digitalocean/godo/storage.go new file mode 100644 index 0000000000000..e1dda598a5197 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/storage.go @@ -0,0 +1,261 @@ +package godo + +import ( + "context" + "fmt" + "net/http" + "time" +) + +const ( + storageBasePath = "v2" + storageAllocPath = storageBasePath + "/volumes" + storageSnapPath = storageBasePath + "/snapshots" +) + +// StorageService is an interface for interfacing with the storage +// endpoints of the Digital Ocean API. +// See: https://developers.digitalocean.com/documentation/v2/#block-storage +type StorageService interface { + ListVolumes(context.Context, *ListVolumeParams) ([]Volume, *Response, error) + GetVolume(context.Context, string) (*Volume, *Response, error) + CreateVolume(context.Context, *VolumeCreateRequest) (*Volume, *Response, error) + DeleteVolume(context.Context, string) (*Response, error) + ListSnapshots(ctx context.Context, volumeID string, opts *ListOptions) ([]Snapshot, *Response, error) + GetSnapshot(context.Context, string) (*Snapshot, *Response, error) + CreateSnapshot(context.Context, *SnapshotCreateRequest) (*Snapshot, *Response, error) + DeleteSnapshot(context.Context, string) (*Response, error) +} + +// StorageServiceOp handles communication with the storage volumes related methods of the +// DigitalOcean API. +type StorageServiceOp struct { + client *Client +} + +// ListVolumeParams stores the options you can set for a ListVolumeCall +type ListVolumeParams struct { + Region string `json:"region"` + Name string `json:"name"` + ListOptions *ListOptions `json:"list_options,omitempty"` +} + +var _ StorageService = &StorageServiceOp{} + +// Volume represents a Digital Ocean block store volume. +type Volume struct { + ID string `json:"id"` + Region *Region `json:"region"` + Name string `json:"name"` + SizeGigaBytes int64 `json:"size_gigabytes"` + Description string `json:"description"` + DropletIDs []int `json:"droplet_ids"` + CreatedAt time.Time `json:"created_at"` + FilesystemType string `json:"filesystem_type"` + FilesystemLabel string `json:"filesystem_label"` + Tags []string `json:"tags"` +} + +func (f Volume) String() string { + return Stringify(f) +} + +func (f Volume) URN() string { + return ToURN("Volume", f.ID) +} + +type storageVolumesRoot struct { + Volumes []Volume `json:"volumes"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +type storageVolumeRoot struct { + Volume *Volume `json:"volume"` + Links *Links `json:"links,omitempty"` +} + +// VolumeCreateRequest represents a request to create a block store +// volume. +type VolumeCreateRequest struct { + Region string `json:"region"` + Name string `json:"name"` + Description string `json:"description"` + SizeGigaBytes int64 `json:"size_gigabytes"` + SnapshotID string `json:"snapshot_id"` + FilesystemType string `json:"filesystem_type"` + FilesystemLabel string `json:"filesystem_label"` + Tags []string `json:"tags"` +} + +// ListVolumes lists all storage volumes. +func (svc *StorageServiceOp) ListVolumes(ctx context.Context, params *ListVolumeParams) ([]Volume, *Response, error) { + path := storageAllocPath + if params != nil { + if params.Region != "" && params.Name != "" { + path = fmt.Sprintf("%s?name=%s®ion=%s", path, params.Name, params.Region) + } else if params.Region != "" { + path = fmt.Sprintf("%s?region=%s", path, params.Region) + } else if params.Name != "" { + path = fmt.Sprintf("%s?name=%s", path, params.Name) + } + + if params.ListOptions != nil { + var err error + path, err = addOptions(path, params.ListOptions) + if err != nil { + return nil, nil, err + } + } + } + + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(storageVolumesRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Volumes, resp, nil +} + +// CreateVolume creates a storage volume. The name must be unique. +func (svc *StorageServiceOp) CreateVolume(ctx context.Context, createRequest *VolumeCreateRequest) (*Volume, *Response, error) { + path := storageAllocPath + + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, createRequest) + if err != nil { + return nil, nil, err + } + + root := new(storageVolumeRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Volume, resp, nil +} + +// GetVolume retrieves an individual storage volume. +func (svc *StorageServiceOp) GetVolume(ctx context.Context, id string) (*Volume, *Response, error) { + path := fmt.Sprintf("%s/%s", storageAllocPath, id) + + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(storageVolumeRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Volume, resp, nil +} + +// DeleteVolume deletes a storage volume. +func (svc *StorageServiceOp) DeleteVolume(ctx context.Context, id string) (*Response, error) { + path := fmt.Sprintf("%s/%s", storageAllocPath, id) + + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + return svc.client.Do(ctx, req, nil) +} + +// SnapshotCreateRequest represents a request to create a block store +// volume. +type SnapshotCreateRequest struct { + VolumeID string `json:"volume_id"` + Name string `json:"name"` + Description string `json:"description"` + Tags []string `json:"tags"` +} + +// ListSnapshots lists all snapshots related to a storage volume. +func (svc *StorageServiceOp) ListSnapshots(ctx context.Context, volumeID string, opt *ListOptions) ([]Snapshot, *Response, error) { + path := fmt.Sprintf("%s/%s/snapshots", storageAllocPath, volumeID) + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(snapshotsRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Snapshots, resp, nil +} + +// CreateSnapshot creates a snapshot of a storage volume. +func (svc *StorageServiceOp) CreateSnapshot(ctx context.Context, createRequest *SnapshotCreateRequest) (*Snapshot, *Response, error) { + path := fmt.Sprintf("%s/%s/snapshots", storageAllocPath, createRequest.VolumeID) + + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, createRequest) + if err != nil { + return nil, nil, err + } + + root := new(snapshotRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Snapshot, resp, nil +} + +// GetSnapshot retrieves an individual snapshot. +func (svc *StorageServiceOp) GetSnapshot(ctx context.Context, id string) (*Snapshot, *Response, error) { + path := fmt.Sprintf("%s/%s", storageSnapPath, id) + + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(snapshotRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Snapshot, resp, nil +} + +// DeleteSnapshot deletes a snapshot. +func (svc *StorageServiceOp) DeleteSnapshot(ctx context.Context, id string) (*Response, error) { + path := fmt.Sprintf("%s/%s", storageSnapPath, id) + + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + return svc.client.Do(ctx, req, nil) +} diff --git a/vendor/github.com/digitalocean/godo/storage_actions.go b/vendor/github.com/digitalocean/godo/storage_actions.go new file mode 100644 index 0000000000000..234aba9064123 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/storage_actions.go @@ -0,0 +1,132 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +// StorageActionsService is an interface for interfacing with the +// storage actions endpoints of the Digital Ocean API. +// See: https://developers.digitalocean.com/documentation/v2#storage-actions +type StorageActionsService interface { + Attach(ctx context.Context, volumeID string, dropletID int) (*Action, *Response, error) + DetachByDropletID(ctx context.Context, volumeID string, dropletID int) (*Action, *Response, error) + Get(ctx context.Context, volumeID string, actionID int) (*Action, *Response, error) + List(ctx context.Context, volumeID string, opt *ListOptions) ([]Action, *Response, error) + Resize(ctx context.Context, volumeID string, sizeGigabytes int, regionSlug string) (*Action, *Response, error) +} + +// StorageActionsServiceOp handles communication with the storage volumes +// action related methods of the DigitalOcean API. +type StorageActionsServiceOp struct { + client *Client +} + +// StorageAttachment represents the attachement of a block storage +// volume to a specific Droplet under the device name. +type StorageAttachment struct { + DropletID int `json:"droplet_id"` +} + +// Attach a storage volume to a Droplet. +func (s *StorageActionsServiceOp) Attach(ctx context.Context, volumeID string, dropletID int) (*Action, *Response, error) { + request := &ActionRequest{ + "type": "attach", + "droplet_id": dropletID, + } + return s.doAction(ctx, volumeID, request) +} + +// DetachByDropletID a storage volume from a Droplet by Droplet ID. +func (s *StorageActionsServiceOp) DetachByDropletID(ctx context.Context, volumeID string, dropletID int) (*Action, *Response, error) { + request := &ActionRequest{ + "type": "detach", + "droplet_id": dropletID, + } + return s.doAction(ctx, volumeID, request) +} + +// Get an action for a particular storage volume by id. +func (s *StorageActionsServiceOp) Get(ctx context.Context, volumeID string, actionID int) (*Action, *Response, error) { + path := fmt.Sprintf("%s/%d", storageAllocationActionPath(volumeID), actionID) + return s.get(ctx, path) +} + +// List the actions for a particular storage volume. +func (s *StorageActionsServiceOp) List(ctx context.Context, volumeID string, opt *ListOptions) ([]Action, *Response, error) { + path := storageAllocationActionPath(volumeID) + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + return s.list(ctx, path) +} + +// Resize a storage volume. +func (s *StorageActionsServiceOp) Resize(ctx context.Context, volumeID string, sizeGigabytes int, regionSlug string) (*Action, *Response, error) { + request := &ActionRequest{ + "type": "resize", + "size_gigabytes": sizeGigabytes, + "region": regionSlug, + } + return s.doAction(ctx, volumeID, request) +} + +func (s *StorageActionsServiceOp) doAction(ctx context.Context, volumeID string, request *ActionRequest) (*Action, *Response, error) { + path := storageAllocationActionPath(volumeID) + + req, err := s.client.NewRequest(ctx, http.MethodPost, path, request) + if err != nil { + return nil, nil, err + } + + root := new(actionRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Event, resp, err +} + +func (s *StorageActionsServiceOp) get(ctx context.Context, path string) (*Action, *Response, error) { + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(actionRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Event, resp, err +} + +func (s *StorageActionsServiceOp) list(ctx context.Context, path string) ([]Action, *Response, error) { + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(actionsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Actions, resp, err +} + +func storageAllocationActionPath(volumeID string) string { + return fmt.Sprintf("%s/%s/actions", storageAllocPath, volumeID) +} diff --git a/vendor/github.com/digitalocean/godo/strings.go b/vendor/github.com/digitalocean/godo/strings.go new file mode 100644 index 0000000000000..4d5c0ad220795 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/strings.go @@ -0,0 +1,102 @@ +package godo + +import ( + "bytes" + "fmt" + "io" + "reflect" + "strings" +) + +var timestampType = reflect.TypeOf(Timestamp{}) + +type ResourceWithURN interface { + URN() string +} + +// ToURN converts the resource type and ID to a valid DO API URN. +func ToURN(resourceType string, id interface{}) string { + return fmt.Sprintf("%s:%s:%v", "do", strings.ToLower(resourceType), id) +} + +// Stringify attempts to create a string representation of DigitalOcean types +func Stringify(message interface{}) string { + var buf bytes.Buffer + v := reflect.ValueOf(message) + stringifyValue(&buf, v) + return buf.String() +} + +// stringifyValue was graciously cargoculted from the goprotubuf library +func stringifyValue(w io.Writer, val reflect.Value) { + if val.Kind() == reflect.Ptr && val.IsNil() { + _, _ = w.Write([]byte("")) + return + } + + v := reflect.Indirect(val) + + switch v.Kind() { + case reflect.String: + fmt.Fprintf(w, `"%s"`, v) + case reflect.Slice: + stringifySlice(w, v) + return + case reflect.Struct: + stringifyStruct(w, v) + default: + if v.CanInterface() { + fmt.Fprint(w, v.Interface()) + } + } +} + +func stringifySlice(w io.Writer, v reflect.Value) { + _, _ = w.Write([]byte{'['}) + for i := 0; i < v.Len(); i++ { + if i > 0 { + _, _ = w.Write([]byte{' '}) + } + + stringifyValue(w, v.Index(i)) + } + + _, _ = w.Write([]byte{']'}) +} + +func stringifyStruct(w io.Writer, v reflect.Value) { + if v.Type().Name() != "" { + _, _ = w.Write([]byte(v.Type().String())) + } + + // special handling of Timestamp values + if v.Type() == timestampType { + fmt.Fprintf(w, "{%s}", v.Interface()) + return + } + + _, _ = w.Write([]byte{'{'}) + + var sep bool + for i := 0; i < v.NumField(); i++ { + fv := v.Field(i) + if fv.Kind() == reflect.Ptr && fv.IsNil() { + continue + } + if fv.Kind() == reflect.Slice && fv.IsNil() { + continue + } + + if sep { + _, _ = w.Write([]byte(", ")) + } else { + sep = true + } + + _, _ = w.Write([]byte(v.Type().Field(i).Name)) + _, _ = w.Write([]byte{':'}) + stringifyValue(w, fv) + } + + _, _ = w.Write([]byte{'}'}) +} diff --git a/vendor/github.com/digitalocean/godo/tags.go b/vendor/github.com/digitalocean/godo/tags.go new file mode 100644 index 0000000000000..6301e15f1c4b7 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/tags.go @@ -0,0 +1,247 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +const tagsBasePath = "v2/tags" + +// TagsService is an interface for interfacing with the tags +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2#tags +type TagsService interface { + List(context.Context, *ListOptions) ([]Tag, *Response, error) + Get(context.Context, string) (*Tag, *Response, error) + Create(context.Context, *TagCreateRequest) (*Tag, *Response, error) + Delete(context.Context, string) (*Response, error) + + TagResources(context.Context, string, *TagResourcesRequest) (*Response, error) + UntagResources(context.Context, string, *UntagResourcesRequest) (*Response, error) +} + +// TagsServiceOp handles communication with tag related method of the +// DigitalOcean API. +type TagsServiceOp struct { + client *Client +} + +var _ TagsService = &TagsServiceOp{} + +// ResourceType represents a class of resource, currently only droplet are supported +type ResourceType string + +const ( + // DropletResourceType holds the string representing our ResourceType of Droplet. + DropletResourceType ResourceType = "droplet" + // ImageResourceType holds the string representing our ResourceType of Image. + ImageResourceType ResourceType = "image" + // VolumeResourceType holds the string representing our ResourceType of Volume. + VolumeResourceType ResourceType = "volume" + // LoadBalancerResourceType holds the string representing our ResourceType of LoadBalancer. + LoadBalancerResourceType ResourceType = "load_balancer" + // VolumeSnapshotResourceType holds the string representing our ResourceType for storage Snapshots. + VolumeSnapshotResourceType ResourceType = "volume_snapshot" + // DatabaseResourceType holds the string representing our ResourceType of Database. + DatabaseResourceType ResourceType = "database" +) + +// Resource represent a single resource for associating/disassociating with tags +type Resource struct { + ID string `json:"resource_id,omitempty"` + Type ResourceType `json:"resource_type,omitempty"` +} + +// TaggedResources represent the set of resources a tag is attached to +type TaggedResources struct { + Count int `json:"count"` + LastTaggedURI string `json:"last_tagged_uri,omitempty"` + Droplets *TaggedDropletsResources `json:"droplets,omitempty"` + Images *TaggedImagesResources `json:"images"` + Volumes *TaggedVolumesResources `json:"volumes"` + VolumeSnapshots *TaggedVolumeSnapshotsResources `json:"volume_snapshots"` + Databases *TaggedDatabasesResources `json:"databases"` +} + +// TaggedDropletsResources represent the droplet resources a tag is attached to +type TaggedDropletsResources struct { + Count int `json:"count,float64,omitempty"` + LastTagged *Droplet `json:"last_tagged,omitempty"` + LastTaggedURI string `json:"last_tagged_uri,omitempty"` +} + +// TaggedResourcesData represent the generic resources a tag is attached to +type TaggedResourcesData struct { + Count int `json:"count,float64,omitempty"` + LastTaggedURI string `json:"last_tagged_uri,omitempty"` +} + +// TaggedImagesResources represent the image resources a tag is attached to +type TaggedImagesResources TaggedResourcesData + +// TaggedVolumesResources represent the volume resources a tag is attached to +type TaggedVolumesResources TaggedResourcesData + +// TaggedVolumeSnapshotsResources represent the volume snapshot resources a tag is attached to +type TaggedVolumeSnapshotsResources TaggedResourcesData + +// TaggedDatabasesResources represent the database resources a tag is attached to +type TaggedDatabasesResources TaggedResourcesData + +// Tag represent DigitalOcean tag +type Tag struct { + Name string `json:"name,omitempty"` + Resources *TaggedResources `json:"resources,omitempty"` +} + +//TagCreateRequest represents the JSON structure of a request of that type. +type TagCreateRequest struct { + Name string `json:"name"` +} + +// TagResourcesRequest represents the JSON structure of a request of that type. +type TagResourcesRequest struct { + Resources []Resource `json:"resources"` +} + +// UntagResourcesRequest represents the JSON structure of a request of that type. +type UntagResourcesRequest struct { + Resources []Resource `json:"resources"` +} + +type tagsRoot struct { + Tags []Tag `json:"tags"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +type tagRoot struct { + Tag *Tag `json:"tag"` +} + +// List all tags +func (s *TagsServiceOp) List(ctx context.Context, opt *ListOptions) ([]Tag, *Response, error) { + path := tagsBasePath + path, err := addOptions(path, opt) + + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(tagsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Tags, resp, err +} + +// Get a single tag +func (s *TagsServiceOp) Get(ctx context.Context, name string) (*Tag, *Response, error) { + path := fmt.Sprintf("%s/%s", tagsBasePath, name) + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(tagRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Tag, resp, err +} + +// Create a new tag +func (s *TagsServiceOp) Create(ctx context.Context, createRequest *TagCreateRequest) (*Tag, *Response, error) { + if createRequest == nil { + return nil, nil, NewArgError("createRequest", "cannot be nil") + } + + req, err := s.client.NewRequest(ctx, http.MethodPost, tagsBasePath, createRequest) + if err != nil { + return nil, nil, err + } + + root := new(tagRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Tag, resp, err +} + +// Delete an existing tag +func (s *TagsServiceOp) Delete(ctx context.Context, name string) (*Response, error) { + if name == "" { + return nil, NewArgError("name", "cannot be empty") + } + + path := fmt.Sprintf("%s/%s", tagsBasePath, name) + req, err := s.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + resp, err := s.client.Do(ctx, req, nil) + + return resp, err +} + +// TagResources associates resources with a given Tag. +func (s *TagsServiceOp) TagResources(ctx context.Context, name string, tagRequest *TagResourcesRequest) (*Response, error) { + if name == "" { + return nil, NewArgError("name", "cannot be empty") + } + + if tagRequest == nil { + return nil, NewArgError("tagRequest", "cannot be nil") + } + + path := fmt.Sprintf("%s/%s/resources", tagsBasePath, name) + req, err := s.client.NewRequest(ctx, http.MethodPost, path, tagRequest) + if err != nil { + return nil, err + } + + resp, err := s.client.Do(ctx, req, nil) + + return resp, err +} + +// UntagResources dissociates resources with a given Tag. +func (s *TagsServiceOp) UntagResources(ctx context.Context, name string, untagRequest *UntagResourcesRequest) (*Response, error) { + if name == "" { + return nil, NewArgError("name", "cannot be empty") + } + + if untagRequest == nil { + return nil, NewArgError("tagRequest", "cannot be nil") + } + + path := fmt.Sprintf("%s/%s/resources", tagsBasePath, name) + req, err := s.client.NewRequest(ctx, http.MethodDelete, path, untagRequest) + if err != nil { + return nil, err + } + + resp, err := s.client.Do(ctx, req, nil) + + return resp, err +} diff --git a/vendor/github.com/digitalocean/godo/timestamp.go b/vendor/github.com/digitalocean/godo/timestamp.go new file mode 100644 index 0000000000000..37a28e5f2f2ec --- /dev/null +++ b/vendor/github.com/digitalocean/godo/timestamp.go @@ -0,0 +1,35 @@ +package godo + +import ( + "strconv" + "time" +) + +// Timestamp represents a time that can be unmarshalled from a JSON string +// formatted as either an RFC3339 or Unix timestamp. All +// exported methods of time.Time can be called on Timestamp. +type Timestamp struct { + time.Time +} + +func (t Timestamp) String() string { + return t.Time.String() +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +// Time is expected in RFC3339 or Unix format. +func (t *Timestamp) UnmarshalJSON(data []byte) error { + str := string(data) + i, err := strconv.ParseInt(str, 10, 64) + if err == nil { + t.Time = time.Unix(i, 0) + } else { + t.Time, err = time.Parse(`"`+time.RFC3339+`"`, str) + } + return err +} + +// Equal reports whether t and u are equal based on time.Equal +func (t Timestamp) Equal(u Timestamp) bool { + return t.Time.Equal(u.Time) +} diff --git a/vendor/github.com/digitalocean/godo/vpcs.go b/vendor/github.com/digitalocean/godo/vpcs.go new file mode 100644 index 0000000000000..7fbeaf88be97a --- /dev/null +++ b/vendor/github.com/digitalocean/godo/vpcs.go @@ -0,0 +1,201 @@ +package godo + +import ( + "context" + "net/http" + "time" +) + +const vpcsBasePath = "/v2/vpcs" + +// VPCsService is an interface for managing Virtual Private Cloud configurations with the +// DigitalOcean API. +// See: https://developers.digitalocean.com/documentation/v2#vpcs +type VPCsService interface { + Create(context.Context, *VPCCreateRequest) (*VPC, *Response, error) + Get(context.Context, string) (*VPC, *Response, error) + List(context.Context, *ListOptions) ([]*VPC, *Response, error) + Update(context.Context, string, *VPCUpdateRequest) (*VPC, *Response, error) + Set(context.Context, string, ...VPCSetField) (*VPC, *Response, error) + Delete(context.Context, string) (*Response, error) +} + +var _ VPCsService = &VPCsServiceOp{} + +// VPCsServiceOp interfaces with VPC endpoints in the DigitalOcean API. +type VPCsServiceOp struct { + client *Client +} + +// VPCCreateRequest represents a request to create a Virtual Private Cloud. +type VPCCreateRequest struct { + Name string `json:"name,omitempty"` + RegionSlug string `json:"region,omitempty"` + Description string `json:"description,omitempty"` + IPRange string `json:"ip_range,omitempty"` +} + +// VPCUpdateRequest represents a request to update a Virtual Private Cloud. +type VPCUpdateRequest struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` +} + +// VPCSetField allows one to set individual fields within a VPC configuration. +type VPCSetField interface { + vpcSetField(map[string]interface{}) +} + +// VPCSetName is used when one want to set the `name` field of a VPC. +// Ex.: VPCs.Set(..., VPCSetName("new-name")) +type VPCSetName string + +// VPCSetDescription is used when one want to set the `description` field of a VPC. +// Ex.: VPCs.Set(..., VPCSetDescription("vpc description")) +type VPCSetDescription string + +// VPC represents a DigitalOcean Virtual Private Cloud configuration. +type VPC struct { + ID string `json:"id,omitempty"` + URN string `json:"urn"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + IPRange string `json:"ip_range,omitempty"` + RegionSlug string `json:"region,omitempty"` + CreatedAt time.Time `json:"created_at,omitempty"` + Default bool `json:"default,omitempty"` +} + +type vpcRoot struct { + VPC *VPC `json:"vpc"` +} + +type vpcsRoot struct { + VPCs []*VPC `json:"vpcs"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +// Get returns the details of a Virtual Private Cloud. +func (v *VPCsServiceOp) Get(ctx context.Context, id string) (*VPC, *Response, error) { + path := vpcsBasePath + "/" + id + req, err := v.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(vpcRoot) + resp, err := v.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.VPC, resp, nil +} + +// Create creates a new Virtual Private Cloud. +func (v *VPCsServiceOp) Create(ctx context.Context, create *VPCCreateRequest) (*VPC, *Response, error) { + path := vpcsBasePath + req, err := v.client.NewRequest(ctx, http.MethodPost, path, create) + if err != nil { + return nil, nil, err + } + + root := new(vpcRoot) + resp, err := v.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.VPC, resp, nil +} + +// List returns a list of the caller's VPCs, with optional pagination. +func (v *VPCsServiceOp) List(ctx context.Context, opt *ListOptions) ([]*VPC, *Response, error) { + path, err := addOptions(vpcsBasePath, opt) + if err != nil { + return nil, nil, err + } + req, err := v.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(vpcsRoot) + resp, err := v.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.VPCs, resp, nil +} + +// Update updates a Virtual Private Cloud's properties. +func (v *VPCsServiceOp) Update(ctx context.Context, id string, update *VPCUpdateRequest) (*VPC, *Response, error) { + path := vpcsBasePath + "/" + id + req, err := v.client.NewRequest(ctx, http.MethodPut, path, update) + if err != nil { + return nil, nil, err + } + + root := new(vpcRoot) + resp, err := v.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.VPC, resp, nil +} + +func (n VPCSetName) vpcSetField(in map[string]interface{}) { + in["name"] = n +} + +func (n VPCSetDescription) vpcSetField(in map[string]interface{}) { + in["description"] = n +} + +// Set updates specific properties of a Virtual Private Cloud. +func (v *VPCsServiceOp) Set(ctx context.Context, id string, fields ...VPCSetField) (*VPC, *Response, error) { + path := vpcsBasePath + "/" + id + update := make(map[string]interface{}, len(fields)) + for _, field := range fields { + field.vpcSetField(update) + } + + req, err := v.client.NewRequest(ctx, http.MethodPatch, path, update) + if err != nil { + return nil, nil, err + } + + root := new(vpcRoot) + resp, err := v.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.VPC, resp, nil +} + +// Delete deletes a Virtual Private Cloud. There is no way to recover a VPC once it has been +// destroyed. +func (v *VPCsServiceOp) Delete(ctx context.Context, id string) (*Response, error) { + path := vpcsBasePath + "/" + id + req, err := v.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + resp, err := v.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + + return resp, nil +} diff --git a/vendor/github.com/google/go-querystring/LICENSE b/vendor/github.com/google/go-querystring/LICENSE new file mode 100644 index 0000000000000..ae121a1e46df9 --- /dev/null +++ b/vendor/github.com/google/go-querystring/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2013 Google. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/google/go-querystring/query/encode.go b/vendor/github.com/google/go-querystring/query/encode.go new file mode 100644 index 0000000000000..37080b19b5d96 --- /dev/null +++ b/vendor/github.com/google/go-querystring/query/encode.go @@ -0,0 +1,320 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package query implements encoding of structs into URL query parameters. +// +// As a simple example: +// +// type Options struct { +// Query string `url:"q"` +// ShowAll bool `url:"all"` +// Page int `url:"page"` +// } +// +// opt := Options{ "foo", true, 2 } +// v, _ := query.Values(opt) +// fmt.Print(v.Encode()) // will output: "q=foo&all=true&page=2" +// +// The exact mapping between Go values and url.Values is described in the +// documentation for the Values() function. +package query + +import ( + "bytes" + "fmt" + "net/url" + "reflect" + "strconv" + "strings" + "time" +) + +var timeType = reflect.TypeOf(time.Time{}) + +var encoderType = reflect.TypeOf(new(Encoder)).Elem() + +// Encoder is an interface implemented by any type that wishes to encode +// itself into URL values in a non-standard way. +type Encoder interface { + EncodeValues(key string, v *url.Values) error +} + +// Values returns the url.Values encoding of v. +// +// Values expects to be passed a struct, and traverses it recursively using the +// following encoding rules. +// +// Each exported struct field is encoded as a URL parameter unless +// +// - the field's tag is "-", or +// - the field is empty and its tag specifies the "omitempty" option +// +// The empty values are false, 0, any nil pointer or interface value, any array +// slice, map, or string of length zero, and any time.Time that returns true +// for IsZero(). +// +// The URL parameter name defaults to the struct field name but can be +// specified in the struct field's tag value. The "url" key in the struct +// field's tag value is the key name, followed by an optional comma and +// options. For example: +// +// // Field is ignored by this package. +// Field int `url:"-"` +// +// // Field appears as URL parameter "myName". +// Field int `url:"myName"` +// +// // Field appears as URL parameter "myName" and the field is omitted if +// // its value is empty +// Field int `url:"myName,omitempty"` +// +// // Field appears as URL parameter "Field" (the default), but the field +// // is skipped if empty. Note the leading comma. +// Field int `url:",omitempty"` +// +// For encoding individual field values, the following type-dependent rules +// apply: +// +// Boolean values default to encoding as the strings "true" or "false". +// Including the "int" option signals that the field should be encoded as the +// strings "1" or "0". +// +// time.Time values default to encoding as RFC3339 timestamps. Including the +// "unix" option signals that the field should be encoded as a Unix time (see +// time.Unix()) +// +// Slice and Array values default to encoding as multiple URL values of the +// same name. Including the "comma" option signals that the field should be +// encoded as a single comma-delimited value. Including the "space" option +// similarly encodes the value as a single space-delimited string. Including +// the "semicolon" option will encode the value as a semicolon-delimited string. +// Including the "brackets" option signals that the multiple URL values should +// have "[]" appended to the value name. "numbered" will append a number to +// the end of each incidence of the value name, example: +// name0=value0&name1=value1, etc. +// +// Anonymous struct fields are usually encoded as if their inner exported +// fields were fields in the outer struct, subject to the standard Go +// visibility rules. An anonymous struct field with a name given in its URL +// tag is treated as having that name, rather than being anonymous. +// +// Non-nil pointer values are encoded as the value pointed to. +// +// Nested structs are encoded including parent fields in value names for +// scoping. e.g: +// +// "user[name]=acme&user[addr][postcode]=1234&user[addr][city]=SFO" +// +// All other values are encoded using their default string representation. +// +// Multiple fields that encode to the same URL parameter name will be included +// as multiple URL values of the same name. +func Values(v interface{}) (url.Values, error) { + values := make(url.Values) + val := reflect.ValueOf(v) + for val.Kind() == reflect.Ptr { + if val.IsNil() { + return values, nil + } + val = val.Elem() + } + + if v == nil { + return values, nil + } + + if val.Kind() != reflect.Struct { + return nil, fmt.Errorf("query: Values() expects struct input. Got %v", val.Kind()) + } + + err := reflectValue(values, val, "") + return values, err +} + +// reflectValue populates the values parameter from the struct fields in val. +// Embedded structs are followed recursively (using the rules defined in the +// Values function documentation) breadth-first. +func reflectValue(values url.Values, val reflect.Value, scope string) error { + var embedded []reflect.Value + + typ := val.Type() + for i := 0; i < typ.NumField(); i++ { + sf := typ.Field(i) + if sf.PkgPath != "" && !sf.Anonymous { // unexported + continue + } + + sv := val.Field(i) + tag := sf.Tag.Get("url") + if tag == "-" { + continue + } + name, opts := parseTag(tag) + if name == "" { + if sf.Anonymous && sv.Kind() == reflect.Struct { + // save embedded struct for later processing + embedded = append(embedded, sv) + continue + } + + name = sf.Name + } + + if scope != "" { + name = scope + "[" + name + "]" + } + + if opts.Contains("omitempty") && isEmptyValue(sv) { + continue + } + + if sv.Type().Implements(encoderType) { + if !reflect.Indirect(sv).IsValid() { + sv = reflect.New(sv.Type().Elem()) + } + + m := sv.Interface().(Encoder) + if err := m.EncodeValues(name, &values); err != nil { + return err + } + continue + } + + if sv.Kind() == reflect.Slice || sv.Kind() == reflect.Array { + var del byte + if opts.Contains("comma") { + del = ',' + } else if opts.Contains("space") { + del = ' ' + } else if opts.Contains("semicolon") { + del = ';' + } else if opts.Contains("brackets") { + name = name + "[]" + } + + if del != 0 { + s := new(bytes.Buffer) + first := true + for i := 0; i < sv.Len(); i++ { + if first { + first = false + } else { + s.WriteByte(del) + } + s.WriteString(valueString(sv.Index(i), opts)) + } + values.Add(name, s.String()) + } else { + for i := 0; i < sv.Len(); i++ { + k := name + if opts.Contains("numbered") { + k = fmt.Sprintf("%s%d", name, i) + } + values.Add(k, valueString(sv.Index(i), opts)) + } + } + continue + } + + for sv.Kind() == reflect.Ptr { + if sv.IsNil() { + break + } + sv = sv.Elem() + } + + if sv.Type() == timeType { + values.Add(name, valueString(sv, opts)) + continue + } + + if sv.Kind() == reflect.Struct { + reflectValue(values, sv, name) + continue + } + + values.Add(name, valueString(sv, opts)) + } + + for _, f := range embedded { + if err := reflectValue(values, f, scope); err != nil { + return err + } + } + + return nil +} + +// valueString returns the string representation of a value. +func valueString(v reflect.Value, opts tagOptions) string { + for v.Kind() == reflect.Ptr { + if v.IsNil() { + return "" + } + v = v.Elem() + } + + if v.Kind() == reflect.Bool && opts.Contains("int") { + if v.Bool() { + return "1" + } + return "0" + } + + if v.Type() == timeType { + t := v.Interface().(time.Time) + if opts.Contains("unix") { + return strconv.FormatInt(t.Unix(), 10) + } + return t.Format(time.RFC3339) + } + + return fmt.Sprint(v.Interface()) +} + +// isEmptyValue checks if a value should be considered empty for the purposes +// of omitting fields with the "omitempty" option. +func isEmptyValue(v reflect.Value) bool { + switch v.Kind() { + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + return v.Len() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Interface, reflect.Ptr: + return v.IsNil() + } + + if v.Type() == timeType { + return v.Interface().(time.Time).IsZero() + } + + return false +} + +// tagOptions is the string following a comma in a struct field's "url" tag, or +// the empty string. It does not include the leading comma. +type tagOptions []string + +// parseTag splits a struct field's url tag into its name and comma-separated +// options. +func parseTag(tag string) (string, tagOptions) { + s := strings.Split(tag, ",") + return s[0], s[1:] +} + +// Contains checks whether the tagOptions contains the specified option. +func (o tagOptions) Contains(option string) bool { + for _, s := range o { + if s == option { + return true + } + } + return false +} diff --git a/vendor/github.com/prometheus/prometheus/NOTICE b/vendor/github.com/prometheus/prometheus/NOTICE index 30ce2a82630e7..5e4f509896b86 100644 --- a/vendor/github.com/prometheus/prometheus/NOTICE +++ b/vendor/github.com/prometheus/prometheus/NOTICE @@ -86,6 +86,11 @@ https://github.com/samuel/go-zookeeper Copyright (c) 2013, Samuel Stauffer See https://github.com/samuel/go-zookeeper/blob/master/LICENSE for license details. +Time series compression algorithm from Facebook's Gorilla paper +https://github.com/dgryski/go-tsz +Copyright (c) 2015,2016 Damian Gryski +See https://github.com/dgryski/go-tsz/blob/master/LICENSE for license details. + We also use code from a large number of npm packages. For details, see: - https://github.com/prometheus/prometheus/blob/master/web/ui/react-app/package.json - https://github.com/prometheus/prometheus/blob/master/web/ui/react-app/package-lock.json diff --git a/vendor/github.com/prometheus/prometheus/discovery/config/config.go b/vendor/github.com/prometheus/prometheus/discovery/config/config.go index 820de1f7c5dd0..1648655700158 100644 --- a/vendor/github.com/prometheus/prometheus/discovery/config/config.go +++ b/vendor/github.com/prometheus/prometheus/discovery/config/config.go @@ -18,6 +18,7 @@ import ( "github.com/prometheus/prometheus/discovery/azure" "github.com/prometheus/prometheus/discovery/consul" + "github.com/prometheus/prometheus/discovery/digitalocean" "github.com/prometheus/prometheus/discovery/dns" "github.com/prometheus/prometheus/discovery/ec2" "github.com/prometheus/prometheus/discovery/file" @@ -40,6 +41,8 @@ type ServiceDiscoveryConfig struct { FileSDConfigs []*file.SDConfig `yaml:"file_sd_configs,omitempty"` // List of Consul service discovery configurations. ConsulSDConfigs []*consul.SDConfig `yaml:"consul_sd_configs,omitempty"` + // List of DigitalOcean service discovery configurations. + DigitalOceanSDConfigs []*digitalocean.SDConfig `yaml:"digitalocean_sd_configs,omitempty"` // List of Serverset service discovery configurations. ServersetSDConfigs []*zookeeper.ServersetSDConfig `yaml:"serverset_sd_configs,omitempty"` // NerveSDConfigs is a list of Nerve service discovery configurations. @@ -72,6 +75,11 @@ func (c *ServiceDiscoveryConfig) Validate() error { return errors.New("empty or null section in consul_sd_configs") } } + for _, cfg := range c.DigitalOceanSDConfigs { + if cfg == nil { + return errors.New("empty or null section in digitalocean_sd_configs") + } + } for _, cfg := range c.DNSSDConfigs { if cfg == nil { return errors.New("empty or null section in dns_sd_configs") diff --git a/vendor/github.com/prometheus/prometheus/discovery/consul/consul.go b/vendor/github.com/prometheus/prometheus/discovery/consul/consul.go index ff1158a0aa93d..684784929e4fa 100644 --- a/vendor/github.com/prometheus/prometheus/discovery/consul/consul.go +++ b/vendor/github.com/prometheus/prometheus/discovery/consul/consul.go @@ -36,7 +36,7 @@ import ( ) const ( - watchTimeout = 30 * time.Second + watchTimeout = 10 * time.Minute retryInterval = 15 * time.Second // addressLabel is the name for the label containing a target's address. @@ -95,7 +95,7 @@ var ( Scheme: "http", Server: "localhost:8500", AllowStale: true, - RefreshInterval: model.Duration(watchTimeout), + RefreshInterval: model.Duration(30 * time.Second), } ) @@ -175,7 +175,7 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) { return nil, err } transport := &http.Transport{ - IdleConnTimeout: 5 * time.Duration(conf.RefreshInterval), + IdleConnTimeout: 2 * time.Duration(watchTimeout), TLSClientConfig: tls, DialContext: conntrack.NewDialContextFunc( conntrack.DialWithTracing(), diff --git a/vendor/github.com/prometheus/prometheus/discovery/digitalocean/digitalocean.go b/vendor/github.com/prometheus/prometheus/discovery/digitalocean/digitalocean.go new file mode 100644 index 0000000000000..dc13ba12c3117 --- /dev/null +++ b/vendor/github.com/prometheus/prometheus/discovery/digitalocean/digitalocean.go @@ -0,0 +1,193 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package digitalocean + +import ( + "context" + "fmt" + "net" + "net/http" + "strconv" + "strings" + "time" + + "github.com/digitalocean/godo" + "github.com/go-kit/kit/log" + config_util "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "github.com/prometheus/common/version" + + "github.com/prometheus/prometheus/discovery/refresh" + "github.com/prometheus/prometheus/discovery/targetgroup" +) + +const ( + doLabel = model.MetaLabelPrefix + "digitalocean_" + doLabelID = doLabel + "droplet_id" + doLabelName = doLabel + "droplet_name" + doLabelImage = doLabel + "image" + doLabelPrivateIPv4 = doLabel + "private_ipv4" + doLabelPublicIPv4 = doLabel + "public_ipv4" + doLabelPublicIPv6 = doLabel + "public_ipv6" + doLabelRegion = doLabel + "region" + doLabelSize = doLabel + "size" + doLabelStatus = doLabel + "status" + doLabelFeatures = doLabel + "features" + doLabelTags = doLabel + "tags" + separator = "," +) + +// DefaultSDConfig is the default DigitalOcean SD configuration. +var DefaultSDConfig = SDConfig{ + Port: 80, + RefreshInterval: model.Duration(60 * time.Second), +} + +// SDConfig is the configuration for DigitalOcean based service discovery. +type SDConfig struct { + HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"` + + RefreshInterval model.Duration `yaml:"refresh_interval"` + Port int `yaml:"port"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultSDConfig + type plain SDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + return nil +} + +// Discovery periodically performs DigitalOcean requests. It implements +// the Discoverer interface. +type Discovery struct { + *refresh.Discovery + client *godo.Client + port int +} + +// NewDiscovery returns a new Discovery which periodically refreshes its targets. +func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) { + d := &Discovery{ + port: conf.Port, + } + + rt, err := config_util.NewRoundTripperFromConfig(conf.HTTPClientConfig, "digitalocean_sd", false) + if err != nil { + return nil, err + } + + d.client, err = godo.New( + &http.Client{ + Transport: rt, + Timeout: time.Duration(conf.RefreshInterval), + }, + godo.SetUserAgent(fmt.Sprintf("Prometheus/%s", version.Version)), + ) + if err != nil { + return nil, fmt.Errorf("error setting up digital ocean agent: %w", err) + } + + d.Discovery = refresh.NewDiscovery( + logger, + "digitalocean", + time.Duration(conf.RefreshInterval), + d.refresh, + ) + return d, nil +} + +func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { + tg := &targetgroup.Group{ + Source: "DigitalOcean", + } + + droplets, err := d.listDroplets() + if err != nil { + return nil, err + } + for _, droplet := range droplets { + if droplet.Networks == nil || len(droplet.Networks.V4) == 0 { + continue + } + + privateIPv4, err := droplet.PrivateIPv4() + if err != nil { + return nil, fmt.Errorf("error while reading private IPv4 of droplet %d: %w", droplet.ID, err) + } + publicIPv4, err := droplet.PublicIPv4() + if err != nil { + return nil, fmt.Errorf("error while reading public IPv4 of droplet %d: %w", droplet.ID, err) + } + publicIPv6, err := droplet.PublicIPv6() + if err != nil { + return nil, fmt.Errorf("error while reading public IPv6 of droplet %d: %w", droplet.ID, err) + } + + labels := model.LabelSet{ + doLabelID: model.LabelValue(fmt.Sprintf("%d", droplet.ID)), + doLabelName: model.LabelValue(droplet.Name), + doLabelImage: model.LabelValue(droplet.Image.Slug), + doLabelPrivateIPv4: model.LabelValue(privateIPv4), + doLabelPublicIPv4: model.LabelValue(publicIPv4), + doLabelPublicIPv6: model.LabelValue(publicIPv6), + doLabelRegion: model.LabelValue(droplet.Region.Slug), + doLabelSize: model.LabelValue(droplet.SizeSlug), + doLabelStatus: model.LabelValue(droplet.Status), + } + + addr := net.JoinHostPort(publicIPv4, strconv.FormatUint(uint64(d.port), 10)) + labels[model.AddressLabel] = model.LabelValue(addr) + + if len(droplet.Features) > 0 { + // We surround the separated list with the separator as well. This way regular expressions + // in relabeling rules don't have to consider feature positions. + features := separator + strings.Join(droplet.Features, separator) + separator + labels[doLabelFeatures] = model.LabelValue(features) + } + + if len(droplet.Tags) > 0 { + // We surround the separated list with the separator as well. This way regular expressions + // in relabeling rules don't have to consider tag positions. + tags := separator + strings.Join(droplet.Tags, separator) + separator + labels[doLabelTags] = model.LabelValue(tags) + } + + tg.Targets = append(tg.Targets, labels) + } + return []*targetgroup.Group{tg}, nil +} + +func (d *Discovery) listDroplets() ([]godo.Droplet, error) { + var ( + droplets []godo.Droplet + opts = &godo.ListOptions{Page: 1} + ) + for { + paginatedDroplets, resp, err := d.client.Droplets.List(context.Background(), opts) + if err != nil { + return nil, fmt.Errorf("error while listing droplets page %d: %w", opts.Page, err) + } + droplets = append(droplets, paginatedDroplets...) + if resp.Links == nil || resp.Links.IsLastPage() { + break + } + opts.Page++ + } + return droplets, nil +} diff --git a/vendor/github.com/prometheus/prometheus/discovery/ec2/ec2.go b/vendor/github.com/prometheus/prometheus/discovery/ec2/ec2.go index 1fdb9cc4e40c3..8d94a0c65ebbc 100644 --- a/vendor/github.com/prometheus/prometheus/discovery/ec2/ec2.go +++ b/vendor/github.com/prometheus/prometheus/discovery/ec2/ec2.go @@ -38,6 +38,7 @@ import ( const ( ec2Label = model.MetaLabelPrefix + "ec2_" + ec2LabelAMI = ec2Label + "ami" ec2LabelAZ = ec2Label + "availability_zone" ec2LabelArch = ec2Label + "architecture" ec2LabelInstanceID = ec2Label + "instance_id" @@ -212,6 +213,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { labels[ec2LabelPublicDNS] = model.LabelValue(*inst.PublicDnsName) } + labels[ec2LabelAMI] = model.LabelValue(*inst.ImageId) labels[ec2LabelAZ] = model.LabelValue(*inst.Placement.AvailabilityZone) labels[ec2LabelInstanceState] = model.LabelValue(*inst.State.Name) labels[ec2LabelInstanceType] = model.LabelValue(*inst.InstanceType) diff --git a/vendor/github.com/prometheus/prometheus/discovery/manager.go b/vendor/github.com/prometheus/prometheus/discovery/manager.go index 66c0057a35cb1..32ef5348dd64e 100644 --- a/vendor/github.com/prometheus/prometheus/discovery/manager.go +++ b/vendor/github.com/prometheus/prometheus/discovery/manager.go @@ -29,6 +29,7 @@ import ( "github.com/prometheus/prometheus/discovery/azure" "github.com/prometheus/prometheus/discovery/consul" + "github.com/prometheus/prometheus/discovery/digitalocean" "github.com/prometheus/prometheus/discovery/dns" "github.com/prometheus/prometheus/discovery/ec2" "github.com/prometheus/prometheus/discovery/file" @@ -369,6 +370,11 @@ func (m *Manager) registerProviders(cfg sd_config.ServiceDiscoveryConfig, setNam return consul.NewDiscovery(c, log.With(m.logger, "discovery", "consul")) }) } + for _, c := range cfg.DigitalOceanSDConfigs { + add(c, func() (Discoverer, error) { + return digitalocean.NewDiscovery(c, log.With(m.logger, "discovery", "digitalocean")) + }) + } for _, c := range cfg.MarathonSDConfigs { add(c, func() (Discoverer, error) { return marathon.NewDiscovery(*c, log.With(m.logger, "discovery", "marathon")) diff --git a/vendor/github.com/prometheus/prometheus/promql/engine.go b/vendor/github.com/prometheus/prometheus/promql/engine.go index c7f02b7ae72ef..bc2cfc1da5879 100644 --- a/vendor/github.com/prometheus/prometheus/promql/engine.go +++ b/vendor/github.com/prometheus/prometheus/promql/engine.go @@ -29,7 +29,7 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" - opentracing "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" @@ -418,7 +418,7 @@ func (ng *Engine) newTestQuery(f func(context.Context) error) Query { // // At this point per query only one EvalStmt is evaluated. Alert and record // statements are not handled by the Engine. -func (ng *Engine) exec(ctx context.Context, q *query) (v parser.Value, w storage.Warnings, err error) { +func (ng *Engine) exec(ctx context.Context, q *query) (v parser.Value, ws storage.Warnings, err error) { ng.metrics.currentQueries.Inc() defer ng.metrics.currentQueries.Dec() @@ -517,13 +517,9 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.Eval } defer querier.Close() - warnings, err := ng.populateSeries(ctxPrepare, querier, s) + ng.populateSeries(querier, s) prepareSpanTimer.Finish() - if err != nil { - return nil, warnings, err - } - evalSpanTimer, ctxInnerEval := query.stats.GetSpanTimer(ctx, stats.InnerEvalTime, ng.metrics.queryInnerEval) // Instant evaluation. This is executed as a range evaluation with one step. if s.Start == s.End && s.Interval == 0 { @@ -539,7 +535,7 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.Eval lookbackDelta: ng.lookbackDelta, } - val, err := evaluator.Eval(s.Expr) + val, warnings, err := evaluator.Eval(s.Expr) if err != nil { return nil, warnings, err } @@ -588,7 +584,7 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.Eval logger: ng.logger, lookbackDelta: ng.lookbackDelta, } - val, err := evaluator.Eval(s.Expr) + val, warnings, err := evaluator.Eval(s.Expr) if err != nil { return nil, warnings, err } @@ -649,35 +645,29 @@ func (ng *Engine) findMinTime(s *parser.EvalStmt) time.Time { return s.Start.Add(-maxOffset) } -func (ng *Engine) populateSeries(ctx context.Context, querier storage.Querier, s *parser.EvalStmt) (storage.Warnings, error) { - var ( - // Whenever a MatrixSelector is evaluated, evalRange is set to the corresponding range. - // The evaluation of the VectorSelector inside then evaluates the given range and unsets - // the variable. - evalRange time.Duration - warnings storage.Warnings - err error - ) +func (ng *Engine) populateSeries(querier storage.Querier, s *parser.EvalStmt) { + // Whenever a MatrixSelector is evaluated, evalRange is set to the corresponding range. + // The evaluation of the VectorSelector inside then evaluates the given range and unsets + // the variable. + var evalRange time.Duration parser.Inspect(s.Expr, func(node parser.Node, path []parser.Node) error { - var set storage.SeriesSet - var wrn storage.Warnings - hints := &storage.SelectHints{ - Start: timestamp.FromTime(s.Start), - End: timestamp.FromTime(s.End), - Step: durationToInt64Millis(s.Interval), - } - - // We need to make sure we select the timerange selected by the subquery. - // TODO(gouthamve): cumulativeSubqueryOffset gives the sum of range and the offset - // we can optimise it by separating out the range and offsets, and subtracting the offsets - // from end also. - subqOffset := ng.cumulativeSubqueryOffset(path) - offsetMilliseconds := durationMilliseconds(subqOffset) - hints.Start = hints.Start - offsetMilliseconds - switch n := node.(type) { case *parser.VectorSelector: + hints := &storage.SelectHints{ + Start: timestamp.FromTime(s.Start), + End: timestamp.FromTime(s.End), + Step: durationToInt64Millis(s.Interval), + } + + // We need to make sure we select the timerange selected by the subquery. + // TODO(gouthamve): cumulativeSubqueryOffset gives the sum of range and the offset + // we can optimise it by separating out the range and offsets, and subtracting the offsets + // from end also. + subqOffset := ng.cumulativeSubqueryOffset(path) + offsetMilliseconds := durationMilliseconds(subqOffset) + hints.Start = hints.Start - offsetMilliseconds + if evalRange == 0 { hints.Start = hints.Start - durationMilliseconds(ng.lookbackDelta) } else { @@ -696,20 +686,12 @@ func (ng *Engine) populateSeries(ctx context.Context, querier storage.Querier, s hints.End = hints.End - offsetMilliseconds } - set, wrn, err = querier.Select(false, hints, n.LabelMatchers...) - warnings = append(warnings, wrn...) - if err != nil { - level.Error(ng.logger).Log("msg", "error selecting series set", "err", err) - return err - } - n.UnexpandedSeriesSet = set - + n.UnexpandedSeriesSet = querier.Select(false, hints, n.LabelMatchers...) case *parser.MatrixSelector: evalRange = n.Range } return nil }) - return warnings, err } // extractFuncFromPath walks up the path and searches for the first instance of @@ -743,34 +725,40 @@ func extractGroupsFromPath(p []parser.Node) (bool, []string) { return false, nil } -func checkForSeriesSetExpansion(ctx context.Context, expr parser.Expr) { +func checkAndExpandSeriesSet(ctx context.Context, expr parser.Expr) (storage.Warnings, error) { switch e := expr.(type) { case *parser.MatrixSelector: - checkForSeriesSetExpansion(ctx, e.VectorSelector) + return checkAndExpandSeriesSet(ctx, e.VectorSelector) case *parser.VectorSelector: - if e.Series == nil { - series, err := expandSeriesSet(ctx, e.UnexpandedSeriesSet) - if err != nil { - panic(err) - } else { - e.Series = series - } + if e.Series != nil { + return nil, nil } + series, ws, err := expandSeriesSet(ctx, e.UnexpandedSeriesSet) + e.Series = series + return ws, err } + return nil, nil } -func expandSeriesSet(ctx context.Context, it storage.SeriesSet) (res []storage.Series, err error) { +func expandSeriesSet(ctx context.Context, it storage.SeriesSet) (res []storage.Series, ws storage.Warnings, err error) { for it.Next() { select { case <-ctx.Done(): - return nil, ctx.Err() + return nil, nil, ctx.Err() default: } res = append(res, it.At()) } - return res, it.Err() + return res, it.Warnings(), it.Err() } +type errWithWarnings struct { + err error + warnings storage.Warnings +} + +func (e errWithWarnings) Error() string { return e.err.Error() } + // An evaluator evaluates given expressions over given fixed timestamps. It // is attached to an engine through which it connects to a querier and reports // errors. On timeout or cancellation of its context it terminates. @@ -799,26 +787,33 @@ func (ev *evaluator) error(err error) { } // recover is the handler that turns panics into returns from the top level of evaluation. -func (ev *evaluator) recover(errp *error) { +func (ev *evaluator) recover(ws *storage.Warnings, errp *error) { e := recover() if e == nil { return } - if err, ok := e.(runtime.Error); ok { + + switch err := e.(type) { + case runtime.Error: // Print the stack trace but do not inhibit the running application. buf := make([]byte, 64<<10) buf = buf[:runtime.Stack(buf, false)] level.Error(ev.logger).Log("msg", "runtime panic in parser", "err", e, "stacktrace", string(buf)) *errp = errors.Wrap(err, "unexpected error") - } else { + case errWithWarnings: + *errp = err.err + *ws = append(*ws, err.warnings...) + default: *errp = e.(error) } } -func (ev *evaluator) Eval(expr parser.Expr) (v parser.Value, err error) { - defer ev.recover(&err) - return ev.eval(expr), nil +func (ev *evaluator) Eval(expr parser.Expr) (v parser.Value, ws storage.Warnings, err error) { + defer ev.recover(&ws, &err) + + v, ws = ev.eval(expr) + return v, ws, nil } // EvalNodeHelper stores extra information and caches for evaluating a single node across steps. @@ -884,17 +879,20 @@ func (enh *EvalNodeHelper) signatureFunc(on bool, names ...string) func(labels.L // the given function with the values computed for each expression at that // step. The return value is the combination into time series of all the // function call results. -func (ev *evaluator) rangeEval(f func([]parser.Value, *EvalNodeHelper) Vector, exprs ...parser.Expr) Matrix { +func (ev *evaluator) rangeEval(f func([]parser.Value, *EvalNodeHelper) (Vector, storage.Warnings), exprs ...parser.Expr) (Matrix, storage.Warnings) { numSteps := int((ev.endTimestamp-ev.startTimestamp)/ev.interval) + 1 matrixes := make([]Matrix, len(exprs)) origMatrixes := make([]Matrix, len(exprs)) originalNumSamples := ev.currentSamples + var warnings storage.Warnings for i, e := range exprs { // Functions will take string arguments from the expressions, not the values. if e != nil && e.Type() != parser.ValueTypeString { // ev.currentSamples will be updated to the correct value within the ev.eval call. - matrixes[i] = ev.eval(e).(Matrix) + val, ws := ev.eval(e) + warnings = append(warnings, ws...) + matrixes[i] = val.(Matrix) // Keep a copy of the original point slices so that they // can be returned to the pool. @@ -946,11 +944,12 @@ func (ev *evaluator) rangeEval(f func([]parser.Value, *EvalNodeHelper) Vector, e } // Make the function call. enh.ts = ts - result := f(args, enh) + result, ws := f(args, enh) if result.ContainsSameLabelset() { ev.errorf("vector cannot contain metrics with the same labelset") } enh.out = result[:0] // Reuse result vector. + warnings = append(warnings, ws...) ev.currentSamples += len(result) // When we reset currentSamples to tempNumSamples during the next iteration of the loop it also @@ -969,7 +968,7 @@ func (ev *evaluator) rangeEval(f func([]parser.Value, *EvalNodeHelper) Vector, e mat[i] = Series{Metric: s.Metric, Points: []Point{s.Point}} } ev.currentSamples = originalNumSamples + mat.TotalSamples() - return mat + return mat, warnings } // Add samples in output vector to output series. @@ -1001,29 +1000,30 @@ func (ev *evaluator) rangeEval(f func([]parser.Value, *EvalNodeHelper) Vector, e mat = append(mat, ss) } ev.currentSamples = originalNumSamples + mat.TotalSamples() - return mat + return mat, warnings } // evalSubquery evaluates given SubqueryExpr and returns an equivalent // evaluated MatrixSelector in its place. Note that the Name and LabelMatchers are not set. -func (ev *evaluator) evalSubquery(subq *parser.SubqueryExpr) *parser.MatrixSelector { - val := ev.eval(subq).(Matrix) +func (ev *evaluator) evalSubquery(subq *parser.SubqueryExpr) (*parser.MatrixSelector, storage.Warnings) { + val, ws := ev.eval(subq) + mat := val.(Matrix) vs := &parser.VectorSelector{ Offset: subq.Offset, - Series: make([]storage.Series, 0, len(val)), + Series: make([]storage.Series, 0, len(mat)), } ms := &parser.MatrixSelector{ Range: subq.Range, VectorSelector: vs, } - for _, s := range val { + for _, s := range mat { vs.Series = append(vs.Series, NewStorageSeries(s)) } - return ms + return ms, ws } // eval evaluates the given expression as the given AST expression node requires. -func (ev *evaluator) eval(expr parser.Expr) parser.Value { +func (ev *evaluator) eval(expr parser.Expr) (parser.Value, storage.Warnings) { // This is the top-level evaluation method. // Thus, we check for timeout/cancellation here. if err := contextDone(ev.ctx, "expression evaluation"); err != nil { @@ -1035,16 +1035,16 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value { case *parser.AggregateExpr: unwrapParenExpr(&e.Param) if s, ok := e.Param.(*parser.StringLiteral); ok { - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { - return ev.aggregation(e.Op, e.Grouping, e.Without, s.Val, v[0].(Vector), enh) + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.aggregation(e.Op, e.Grouping, e.Without, s.Val, v[0].(Vector), enh), nil }, e.Expr) } - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { var param float64 if e.Param != nil { param = v[0].(Vector)[0].V } - return ev.aggregation(e.Op, e.Grouping, e.Without, param, v[1].(Vector), enh) + return ev.aggregation(e.Op, e.Grouping, e.Without, param, v[1].(Vector), enh), nil }, e.Param, e.Expr) case *parser.Call: @@ -1056,15 +1056,19 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value { // a vector selector. vs, ok := e.Args[0].(*parser.VectorSelector) if ok { - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { - return call([]parser.Value{ev.vectorSelector(vs, enh.ts)}, e.Args, enh) + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { + val, ws := ev.vectorSelector(vs, enh.ts) + return call([]parser.Value{val}, e.Args, enh), ws }) } } // Check if the function has a matrix argument. - var matrixArgIndex int - var matrixArg bool + var ( + matrixArgIndex int + matrixArg bool + warnings storage.Warnings + ) for i := range e.Args { unwrapParenExpr(&e.Args[i]) a := e.Args[i] @@ -1078,14 +1082,16 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value { matrixArgIndex = i matrixArg = true // Replacing parser.SubqueryExpr with parser.MatrixSelector. - e.Args[i] = ev.evalSubquery(subq) + val, ws := ev.evalSubquery(subq) + e.Args[i] = val + warnings = append(warnings, ws...) break } } if !matrixArg { // Does not have a matrix argument. - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { - return call(v, e.Args, enh) + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return call(v, e.Args, enh), warnings }, e.Args...) } @@ -1095,16 +1101,22 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value { otherInArgs := make([]Vector, len(e.Args)) for i, e := range e.Args { if i != matrixArgIndex { - otherArgs[i] = ev.eval(e).(Matrix) + val, ws := ev.eval(e) + otherArgs[i] = val.(Matrix) otherInArgs[i] = Vector{Sample{}} inArgs[i] = otherInArgs[i] + warnings = append(warnings, ws...) } } sel := e.Args[matrixArgIndex].(*parser.MatrixSelector) selVS := sel.VectorSelector.(*parser.VectorSelector) - checkForSeriesSetExpansion(ev.ctx, sel) + ws, err := checkAndExpandSeriesSet(ev.ctx, sel) + warnings = append(warnings, ws...) + if err != nil { + ev.error(errWithWarnings{errors.Wrap(err, "expanding series"), warnings}) + } mat := make(Matrix, 0, len(selVS.Series)) // Output matrix. offset := durationMilliseconds(selVS.Offset) selRange := durationMilliseconds(sel.Range) @@ -1182,7 +1194,7 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value { // Iterate once to look for a complete series. for _, s := range mat { if len(s.Points) == steps { - return Matrix{} + return Matrix{}, warnings } } @@ -1193,7 +1205,7 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value { found[p.T] = struct{}{} } if i > 0 && len(found) == steps { - return Matrix{} + return Matrix{}, warnings } } @@ -1209,20 +1221,21 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value { Metric: createLabelsForAbsentFunction(e.Args[0]), Points: newp, }, - } + }, warnings } if mat.ContainsSameLabelset() { ev.errorf("vector cannot contain metrics with the same labelset") } - return mat + return mat, warnings case *parser.ParenExpr: return ev.eval(e.Expr) case *parser.UnaryExpr: - mat := ev.eval(e.Expr).(Matrix) + val, ws := ev.eval(e.Expr) + mat := val.(Matrix) if e.Op == parser.SUB { for i := range mat { mat[i].Metric = dropMetricName(mat[i].Metric) @@ -1234,53 +1247,56 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value { ev.errorf("vector cannot contain metrics with the same labelset") } } - return mat + return mat, ws case *parser.BinaryExpr: switch lt, rt := e.LHS.Type(), e.RHS.Type(); { case lt == parser.ValueTypeScalar && rt == parser.ValueTypeScalar: - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { val := scalarBinop(e.Op, v[0].(Vector)[0].Point.V, v[1].(Vector)[0].Point.V) - return append(enh.out, Sample{Point: Point{V: val}}) + return append(enh.out, Sample{Point: Point{V: val}}), nil }, e.LHS, e.RHS) case lt == parser.ValueTypeVector && rt == parser.ValueTypeVector: switch e.Op { case parser.LAND: - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { - return ev.VectorAnd(v[0].(Vector), v[1].(Vector), e.VectorMatching, enh) + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.VectorAnd(v[0].(Vector), v[1].(Vector), e.VectorMatching, enh), nil }, e.LHS, e.RHS) case parser.LOR: - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { - return ev.VectorOr(v[0].(Vector), v[1].(Vector), e.VectorMatching, enh) + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.VectorOr(v[0].(Vector), v[1].(Vector), e.VectorMatching, enh), nil }, e.LHS, e.RHS) case parser.LUNLESS: - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { - return ev.VectorUnless(v[0].(Vector), v[1].(Vector), e.VectorMatching, enh) + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.VectorUnless(v[0].(Vector), v[1].(Vector), e.VectorMatching, enh), nil }, e.LHS, e.RHS) default: - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { - return ev.VectorBinop(e.Op, v[0].(Vector), v[1].(Vector), e.VectorMatching, e.ReturnBool, enh) + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.VectorBinop(e.Op, v[0].(Vector), v[1].(Vector), e.VectorMatching, e.ReturnBool, enh), nil }, e.LHS, e.RHS) } case lt == parser.ValueTypeVector && rt == parser.ValueTypeScalar: - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { - return ev.VectorscalarBinop(e.Op, v[0].(Vector), Scalar{V: v[1].(Vector)[0].Point.V}, false, e.ReturnBool, enh) + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.VectorscalarBinop(e.Op, v[0].(Vector), Scalar{V: v[1].(Vector)[0].Point.V}, false, e.ReturnBool, enh), nil }, e.LHS, e.RHS) case lt == parser.ValueTypeScalar && rt == parser.ValueTypeVector: - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { - return ev.VectorscalarBinop(e.Op, v[1].(Vector), Scalar{V: v[0].(Vector)[0].Point.V}, true, e.ReturnBool, enh) + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.VectorscalarBinop(e.Op, v[1].(Vector), Scalar{V: v[0].(Vector)[0].Point.V}, true, e.ReturnBool, enh), nil }, e.LHS, e.RHS) } case *parser.NumberLiteral: - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { - return append(enh.out, Sample{Point: Point{V: e.Val}}) + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return append(enh.out, Sample{Point: Point{V: e.Val}}), nil }) case *parser.VectorSelector: - checkForSeriesSetExpansion(ev.ctx, e) + ws, err := checkAndExpandSeriesSet(ev.ctx, e) + if err != nil { + ev.error(errWithWarnings{errors.Wrap(err, "expanding series"), ws}) + } mat := make(Matrix, 0, len(e.Series)) it := storage.NewBuffer(durationMilliseconds(ev.lookbackDelta)) for i, s := range e.Series { @@ -1307,9 +1323,8 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value { } else { putPointSlice(ss.Points) } - } - return mat + return mat, ws case *parser.MatrixSelector: if ev.startTimestamp != ev.endTimestamp { @@ -1342,11 +1357,11 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value { newEv.startTimestamp += newEv.interval } - res := newEv.eval(e.Expr) + res, ws := newEv.eval(e.Expr) ev.currentSamples = newEv.currentSamples - return res + return res, ws case *parser.StringLiteral: - return String{V: e.Val, T: ev.startTimestamp} + return String{V: e.Val, T: ev.startTimestamp}, nil } panic(errors.Errorf("unhandled expression of type: %T", expr)) @@ -1357,13 +1372,12 @@ func durationToInt64Millis(d time.Duration) int64 { } // vectorSelector evaluates a *parser.VectorSelector expression. -func (ev *evaluator) vectorSelector(node *parser.VectorSelector, ts int64) Vector { - checkForSeriesSetExpansion(ev.ctx, node) - - var ( - vec = make(Vector, 0, len(node.Series)) - ) - +func (ev *evaluator) vectorSelector(node *parser.VectorSelector, ts int64) (Vector, storage.Warnings) { + ws, err := checkAndExpandSeriesSet(ev.ctx, node) + if err != nil { + ev.error(errWithWarnings{errors.Wrap(err, "expanding series"), ws}) + } + vec := make(Vector, 0, len(node.Series)) it := storage.NewBuffer(durationMilliseconds(ev.lookbackDelta)) for i, s := range node.Series { it.Reset(s.Iterator()) @@ -1381,7 +1395,7 @@ func (ev *evaluator) vectorSelector(node *parser.VectorSelector, ts int64) Vecto ev.error(ErrTooManySamples(env)) } } - return vec + return vec, ws } // vectorSelectorSingle evaluates a instant vector for the iterator of one time series. @@ -1429,21 +1443,23 @@ func putPointSlice(p []Point) { } // matrixSelector evaluates a *parser.MatrixSelector expression. -func (ev *evaluator) matrixSelector(node *parser.MatrixSelector) Matrix { - checkForSeriesSetExpansion(ev.ctx, node) - - vs := node.VectorSelector.(*parser.VectorSelector) - +func (ev *evaluator) matrixSelector(node *parser.MatrixSelector) (Matrix, storage.Warnings) { var ( + vs = node.VectorSelector.(*parser.VectorSelector) + offset = durationMilliseconds(vs.Offset) maxt = ev.startTimestamp - offset mint = maxt - durationMilliseconds(node.Range) matrix = make(Matrix, 0, len(vs.Series)) + + it = storage.NewBuffer(durationMilliseconds(node.Range)) ) + ws, err := checkAndExpandSeriesSet(ev.ctx, node) + if err != nil { + ev.error(errWithWarnings{errors.Wrap(err, "expanding series"), ws}) + } - it := storage.NewBuffer(durationMilliseconds(node.Range)) series := vs.Series - for i, s := range series { if err := contextDone(ev.ctx, "expression evaluation"); err != nil { ev.error(err) @@ -1461,7 +1477,7 @@ func (ev *evaluator) matrixSelector(node *parser.MatrixSelector) Matrix { putPointSlice(ss.Points) } } - return matrix + return matrix, ws } // matrixIterSlice populates a matrix vector covering the requested range for a diff --git a/vendor/github.com/prometheus/prometheus/promql/quantile.go b/vendor/github.com/prometheus/prometheus/promql/quantile.go index 6e63efab6ff19..a25f8917c98fd 100644 --- a/vendor/github.com/prometheus/prometheus/promql/quantile.go +++ b/vendor/github.com/prometheus/prometheus/promql/quantile.go @@ -168,7 +168,7 @@ func coalesceBuckets(buckets buckets) buckets { func ensureMonotonic(buckets buckets) { max := buckets[0].count - for i := range buckets[1:] { + for i := 1; i < len(buckets); i++ { switch { case buckets[i].count > max: max = buckets[i].count diff --git a/vendor/github.com/prometheus/prometheus/rules/manager.go b/vendor/github.com/prometheus/prometheus/rules/manager.go index 1e549fef67783..f999aa19bc267 100644 --- a/vendor/github.com/prometheus/prometheus/rules/manager.go +++ b/vendor/github.com/prometheus/prometheus/rules/manager.go @@ -707,12 +707,7 @@ func (g *Group) RestoreForState(ts time.Time) { matchers = append(matchers, mt) } - sset, err, _ := q.Select(false, nil, matchers...) - if err != nil { - level.Error(g.logger).Log("msg", "Failed to restore 'for' state", - labels.AlertName, alertRule.Name(), "stage", "Select", "err", err) - return - } + sset := q.Select(false, nil, matchers...) seriesFound := false var s storage.Series @@ -727,6 +722,17 @@ func (g *Group) RestoreForState(ts time.Time) { } } + if err := sset.Err(); err != nil { + // Querier Warnings are ignored. We do not care unless we have an error. + level.Error(g.logger).Log( + "msg", "Failed to restore 'for' state", + labels.AlertName, alertRule.Name(), + "stage", "Select", + "err", err, + ) + return + } + if !seriesFound { return } diff --git a/vendor/github.com/prometheus/prometheus/scrape/scrape.go b/vendor/github.com/prometheus/prometheus/scrape/scrape.go index d1987e97c650f..4c6d3bab00d9c 100644 --- a/vendor/github.com/prometheus/prometheus/scrape/scrape.go +++ b/vendor/github.com/prometheus/prometheus/scrape/scrape.go @@ -1140,7 +1140,7 @@ loop: if ok { err = app.AddFast(ce.ref, t, v) - sampleAdded, err = sl.checkAddError(ce, met, tp, err, &sampleLimitErr, &appErrs) + _, err = sl.checkAddError(ce, met, tp, err, &sampleLimitErr, &appErrs) // In theory this should never happen. if err == storage.ErrNotFound { ok = false @@ -1187,10 +1187,10 @@ loop: } } - // Increment added even if there's a sampleLimitErr so we correctly report the number of samples scraped. - if sampleAdded || sampleLimitErr != nil { - added++ - } + // Increment added even if there's an error so we correctly report the + // number of samples remaining after relabelling. + added++ + } if sampleLimitErr != nil { if err == nil { @@ -1275,7 +1275,7 @@ const ( scrapeSeriesAddedMetricName = "scrape_series_added" + "\xff" ) -func (sl *scrapeLoop) report(start time.Time, duration time.Duration, scraped, appended, seriesAdded int, scrapeErr error) (err error) { +func (sl *scrapeLoop) report(start time.Time, duration time.Duration, scraped, added, seriesAdded int, scrapeErr error) (err error) { sl.scraper.Report(start, duration, scrapeErr) ts := timestamp.FromTime(start) @@ -1302,7 +1302,7 @@ func (sl *scrapeLoop) report(start time.Time, duration time.Duration, scraped, a if err = sl.addReportSample(app, scrapeSamplesMetricName, ts, float64(scraped)); err != nil { return } - if err = sl.addReportSample(app, samplesPostRelabelMetricName, ts, float64(appended)); err != nil { + if err = sl.addReportSample(app, samplesPostRelabelMetricName, ts, float64(added)); err != nil { return } if err = sl.addReportSample(app, scrapeSeriesAddedMetricName, ts, float64(seriesAdded)); err != nil { diff --git a/vendor/github.com/prometheus/prometheus/storage/fanout.go b/vendor/github.com/prometheus/prometheus/storage/fanout.go index 8692078a34506..73df5c8c18425 100644 --- a/vendor/github.com/prometheus/prometheus/storage/fanout.go +++ b/vendor/github.com/prometheus/prometheus/storage/fanout.go @@ -16,14 +16,15 @@ package storage import ( "container/heap" "context" - "reflect" "sort" "strings" + "sync" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" @@ -37,8 +38,15 @@ type fanout struct { secondaries []Storage } -// NewFanout returns a new fan-out Storage, which proxies reads and writes +// NewFanout returns a new fanout Storage, which proxies reads and writes // through to multiple underlying storages. +// +// The difference between primary and secondary Storage is only for read (Querier) path and it goes as follows: +// * If the primary querier returns an error, then any of the Querier operations will fail. +// * If any secondary querier returns an error the result from that queries is discarded. The overall operation will succeed, +// and the error from the secondary querier will be returned as a warning. +// +// NOTE: In the case of Prometheus, it treats all remote storages as secondary / best effort. func NewFanout(logger log.Logger, primary Storage, secondaries ...Storage) Storage { return &fanout{ logger: logger, @@ -56,8 +64,8 @@ func (f *fanout) StartTime() (int64, error) { return int64(model.Latest), err } - for _, storage := range f.secondaries { - t, err := storage.StartTime() + for _, s := range f.secondaries { + t, err := s.StartTime() if err != nil { return int64(model.Latest), err } @@ -69,29 +77,27 @@ func (f *fanout) StartTime() (int64, error) { } func (f *fanout) Querier(ctx context.Context, mint, maxt int64) (Querier, error) { - queriers := make([]Querier, 0, 1+len(f.secondaries)) - - // Add primary querier. - primaryQuerier, err := f.primary.Querier(ctx, mint, maxt) + primary, err := f.primary.Querier(ctx, mint, maxt) if err != nil { return nil, err } - queriers = append(queriers, primaryQuerier) - // Add secondary queriers. + secondaries := make([]Querier, 0, len(f.secondaries)) for _, storage := range f.secondaries { querier, err := storage.Querier(ctx, mint, maxt) if err != nil { - for _, q := range queriers { - // TODO(bwplotka): Log error. - _ = q.Close() + // Close already open Queriers, append potential errors to returned error. + errs := tsdb_errors.MultiError{err} + errs.Add(primary.Close()) + for _, q := range secondaries { + errs.Add(q.Close()) } - return nil, err + return nil, errs.Err() } - queriers = append(queriers, querier) - } - return NewMergeQuerier(primaryQuerier, queriers, ChainedSeriesMerge), nil + secondaries = append(secondaries, querier) + } + return NewMergeQuerier(primary, secondaries, ChainedSeriesMerge), nil } func (f *fanout) Appender() Appender { @@ -109,18 +115,12 @@ func (f *fanout) Appender() Appender { // Close closes the storage and all its underlying resources. func (f *fanout) Close() error { - if err := f.primary.Close(); err != nil { - return err + errs := tsdb_errors.MultiError{} + errs.Add(f.primary.Close()) + for _, s := range f.secondaries { + errs.Add(s.Close()) } - - // TODO return multiple errors? - var lastErr error - for _, storage := range f.secondaries { - if err := storage.Close(); err != nil { - lastErr = err - } - } - return lastErr + return errs.Err() } // fanoutAppender implements Appender. @@ -188,153 +188,138 @@ func (f *fanoutAppender) Rollback() (err error) { } type mergeGenericQuerier struct { - mergeFunc genericSeriesMergeFunc - - primaryQuerier genericQuerier - queriers []genericQuerier - failedQueriers map[genericQuerier]struct{} - setQuerierMap map[genericSeriesSet]genericQuerier -} - -// NewMergeQuerier returns a new Querier that merges results of chkQuerierSeries queriers. -// NewMergeQuerier will return NoopQuerier if no queriers are passed to it -// and will filter NoopQueriers from its arguments, in order to reduce overhead -// when only one querier is passed. -// The difference between primary and secondary is as follows: f the primaryQuerier returns an error, query fails. -// For secondaries it just return warnings. -func NewMergeQuerier(primaryQuerier Querier, queriers []Querier, mergeFunc VerticalSeriesMergeFunc) Querier { - filtered := make([]genericQuerier, 0, len(queriers)) - for _, querier := range queriers { - if _, ok := querier.(noopQuerier); !ok && querier != nil { - filtered = append(filtered, newGenericQuerierFrom(querier)) - } - } + queriers []genericQuerier - if len(filtered) == 0 { - return primaryQuerier - } + // mergeFn is used when we see series from different queriers Selects with the same labels. + mergeFn genericSeriesMergeFunc +} - if primaryQuerier == nil && len(filtered) == 1 { - return &querierAdapter{filtered[0]} +// NewMergeQuerier returns a new Querier that merges results of given primary and slice of secondary queriers. +// See NewFanout commentary to learn more about primary vs secondary differences. +// +// In case of overlaps between the data given by primary + secondaries Selects, merge function will be used. +func NewMergeQuerier(primary Querier, secondaries []Querier, mergeFn VerticalSeriesMergeFunc) Querier { + queriers := make([]genericQuerier, 0, len(secondaries)+1) + if primary != nil { + queriers = append(queriers, newGenericQuerierFrom(primary)) + } + for _, querier := range secondaries { + if _, ok := querier.(noopQuerier); !ok && querier != nil { + queriers = append(queriers, newSecondaryQuerierFrom(querier)) + } } return &querierAdapter{&mergeGenericQuerier{ - mergeFunc: (&seriesMergerAdapter{VerticalSeriesMergeFunc: mergeFunc}).Merge, - primaryQuerier: newGenericQuerierFrom(primaryQuerier), - queriers: filtered, - failedQueriers: make(map[genericQuerier]struct{}), - setQuerierMap: make(map[genericSeriesSet]genericQuerier), + mergeFn: (&seriesMergerAdapter{VerticalSeriesMergeFunc: mergeFn}).Merge, + queriers: queriers, }} } -// NewMergeChunkQuerier returns a new ChunkQuerier that merges results of chkQuerierSeries chunk queriers. -// NewMergeChunkQuerier will return NoopChunkQuerier if no chunk queriers are passed to it, -// and will filter NoopQuerieNoopChunkQuerierrs from its arguments, in order to reduce overhead -// when only one chunk querier is passed. -func NewMergeChunkQuerier(primaryQuerier ChunkQuerier, queriers []ChunkQuerier, merger VerticalChunkSeriesMergerFunc) ChunkQuerier { - filtered := make([]genericQuerier, 0, len(queriers)) - for _, querier := range queriers { +// NewMergeChunkQuerier returns a new ChunkQuerier that merges results of given primary and slice of secondary chunk queriers. +// See NewFanout commentary to learn more about primary vs secondary differences. +// +// In case of overlaps between the data given by primary + secondaries Selects, merge function will be used. +// TODO(bwplotka): Currently merge will compact overlapping chunks with bigger chunk, without limit. Split it: https://github.com/prometheus/tsdb/issues/670 +func NewMergeChunkQuerier(primary ChunkQuerier, secondaries []ChunkQuerier, mergeFn VerticalChunkSeriesMergerFunc) ChunkQuerier { + queriers := make([]genericQuerier, 0, len(secondaries)+1) + if primary != nil { + queriers = append(queriers, newGenericQuerierFromChunk(primary)) + } + for _, querier := range secondaries { if _, ok := querier.(noopChunkQuerier); !ok && querier != nil { - filtered = append(filtered, newGenericQuerierFromChunk(querier)) + queriers = append(queriers, newSecondaryQuerierFromChunk(querier)) } } - if len(filtered) == 0 { - return primaryQuerier - } - - if primaryQuerier == nil && len(filtered) == 1 { - return &chunkQuerierAdapter{filtered[0]} - } - return &chunkQuerierAdapter{&mergeGenericQuerier{ - mergeFunc: (&chunkSeriesMergerAdapter{VerticalChunkSeriesMergerFunc: merger}).Merge, - primaryQuerier: newGenericQuerierFromChunk(primaryQuerier), - queriers: filtered, - failedQueriers: make(map[genericQuerier]struct{}), - setQuerierMap: make(map[genericSeriesSet]genericQuerier), + mergeFn: (&chunkSeriesMergerAdapter{VerticalChunkSeriesMergerFunc: mergeFn}).Merge, + queriers: queriers, }} } // Select returns a set of series that matches the given label matchers. -func (q *mergeGenericQuerier) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) (genericSeriesSet, Warnings, error) { +func (q *mergeGenericQuerier) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) genericSeriesSet { if len(q.queriers) == 1 { return q.queriers[0].Select(sortSeries, hints, matchers...) } var ( - seriesSets = make([]genericSeriesSet, 0, len(q.queriers)) - warnings Warnings - priErr error + seriesSets = make([]genericSeriesSet, 0, len(q.queriers)) + wg sync.WaitGroup + seriesSetChan = make(chan genericSeriesSet) ) - type queryResult struct { - qr genericQuerier - set genericSeriesSet - wrn Warnings - selectError error - } - queryResultChan := make(chan *queryResult) + // Schedule all Selects for all queriers we know about. for _, querier := range q.queriers { + wg.Add(1) go func(qr genericQuerier) { + defer wg.Done() + // We need to sort for NewMergeSeriesSet to work. - set, wrn, err := qr.Select(true, hints, matchers...) - queryResultChan <- &queryResult{qr: qr, set: set, wrn: wrn, selectError: err} + seriesSetChan <- qr.Select(true, hints, matchers...) }(querier) } - for i := 0; i < len(q.queriers); i++ { - qryResult := <-queryResultChan - q.setQuerierMap[qryResult.set] = qryResult.qr - if qryResult.wrn != nil { - warnings = append(warnings, qryResult.wrn...) - } - if qryResult.selectError != nil { - q.failedQueriers[qryResult.qr] = struct{}{} - // If the error source isn't the primary querier, return the error as a warning and continue. - if !reflect.DeepEqual(qryResult.qr, q.primaryQuerier) { - warnings = append(warnings, qryResult.selectError) - } else { - priErr = qryResult.selectError + go func() { + wg.Wait() + close(seriesSetChan) + }() + + for r := range seriesSetChan { + seriesSets = append(seriesSets, r) + } + return &lazySeriesSet{create: create(seriesSets, q.mergeFn)} +} + +func create(seriesSets []genericSeriesSet, mergeFn genericSeriesMergeFunc) func() (genericSeriesSet, bool) { + // Returned function gets called with the first call to Next(). + return func() (genericSeriesSet, bool) { + if len(seriesSets) == 1 { + return seriesSets[0], seriesSets[0].Next() + } + var h genericSeriesSetHeap + for _, set := range seriesSets { + if set == nil { + continue + } + if set.Next() { + heap.Push(&h, set) + continue + } + // When primary fails ignore results from secondaries. + // Only the primary querier returns error. + if err := set.Err(); err != nil { + return errorOnlySeriesSet{err}, false } - continue } - seriesSets = append(seriesSets, qryResult.set) - } - if priErr != nil { - return nil, nil, priErr + set := &genericMergeSeriesSet{ + mergeFn: mergeFn, + sets: seriesSets, + heap: h, + } + return set, set.Next() } - return newGenericMergeSeriesSet(seriesSets, q, q.mergeFunc), warnings, nil } // LabelValues returns all potential values for a label name. func (q *mergeGenericQuerier) LabelValues(name string) ([]string, Warnings, error) { - var results [][]string - var warnings Warnings + var ( + results [][]string + warnings Warnings + ) for _, querier := range q.queriers { values, wrn, err := querier.LabelValues(name) if wrn != nil { + // TODO(bwplotka): We could potentially wrap warnings. warnings = append(warnings, wrn...) } if err != nil { - q.failedQueriers[querier] = struct{}{} - // If the error source isn't the primary querier, return the error as a warning and continue. - if querier != q.primaryQuerier { - warnings = append(warnings, err) - continue - } else { - return nil, nil, err - } + return nil, nil, errors.Wrapf(err, "LabelValues() from Querier for label %s", name) } results = append(results, values) } return mergeStringSlices(results), warnings, nil } -func (q *mergeGenericQuerier) IsFailedSet(set genericSeriesSet) bool { - _, isFailedQuerier := q.failedQueriers[q.setQuerierMap[set]] - return isFailedQuerier -} - func mergeStringSlices(ss [][]string) []string { switch len(ss) { case 0: @@ -381,37 +366,31 @@ func (q *mergeGenericQuerier) LabelNames() ([]string, Warnings, error) { for _, querier := range q.queriers { names, wrn, err := querier.LabelNames() if wrn != nil { + // TODO(bwplotka): We could potentially wrap warnings. warnings = append(warnings, wrn...) } - if err != nil { - q.failedQueriers[querier] = struct{}{} - // If the error source isn't the primaryQuerier querier, return the error as a warning and continue. - if querier != q.primaryQuerier { - warnings = append(warnings, err) - continue - } else { - return nil, nil, errors.Wrap(err, "LabelNames() from Querier") - } + return nil, nil, errors.Wrap(err, "LabelNames() from Querier") } - for _, name := range names { labelNamesMap[name] = struct{}{} } } + if len(labelNamesMap) == 0 { + return nil, warnings, nil + } labelNames := make([]string, 0, len(labelNamesMap)) for name := range labelNamesMap { labelNames = append(labelNames, name) } sort.Strings(labelNames) - return labelNames, warnings, nil } // Close releases the resources of the Querier. func (q *mergeGenericQuerier) Close() error { - var errs tsdb_errors.MultiError + errs := tsdb_errors.MultiError{} for _, querier := range q.queriers { if err := querier.Close(); err != nil { errs.Add(err) @@ -420,18 +399,6 @@ func (q *mergeGenericQuerier) Close() error { return errs.Err() } -// genericMergeSeriesSet implements genericSeriesSet -type genericMergeSeriesSet struct { - currentLabels labels.Labels - mergeFunc genericSeriesMergeFunc - - heap genericSeriesSetHeap - sets []genericSeriesSet - - currentSets []genericSeriesSet - querier *mergeGenericQuerier -} - // VerticalSeriesMergeFunc returns merged series implementation that merges series with same labels together. // It has to handle time-overlapped series as well. type VerticalSeriesMergeFunc func(...Series) Series @@ -447,7 +414,7 @@ func NewMergeSeriesSet(sets []SeriesSet, merger VerticalSeriesMergeFunc) SeriesS genericSets = append(genericSets, &genericSeriesSetAdapter{s}) } - return &seriesSetAdapter{newGenericMergeSeriesSet(genericSets, nil, (&seriesMergerAdapter{VerticalSeriesMergeFunc: merger}).Merge)} + return &seriesSetAdapter{newGenericMergeSeriesSet(genericSets, (&seriesMergerAdapter{VerticalSeriesMergeFunc: merger}).Merge)} } // NewMergeChunkSeriesSet returns a new ChunkSeriesSet that merges results of chkQuerierSeries ChunkSeriesSets. @@ -457,21 +424,30 @@ func NewMergeChunkSeriesSet(sets []ChunkSeriesSet, merger VerticalChunkSeriesMer genericSets = append(genericSets, &genericChunkSeriesSetAdapter{s}) } - return &chunkSeriesSetAdapter{newGenericMergeSeriesSet(genericSets, nil, (&chunkSeriesMergerAdapter{VerticalChunkSeriesMergerFunc: merger}).Merge)} + return &chunkSeriesSetAdapter{newGenericMergeSeriesSet(genericSets, (&chunkSeriesMergerAdapter{VerticalChunkSeriesMergerFunc: merger}).Merge)} +} + +// genericMergeSeriesSet implements genericSeriesSet. +type genericMergeSeriesSet struct { + currentLabels labels.Labels + mergeFn genericSeriesMergeFunc + + heap genericSeriesSetHeap + sets []genericSeriesSet + currentSets []genericSeriesSet } // newGenericMergeSeriesSet returns a new genericSeriesSet that merges (and deduplicates) -// series returned by the chkQuerierSeries series sets when iterating. -// Each chkQuerierSeries series set must return its series in labels order, otherwise +// series returned by the series sets when iterating. +// Each series set must return its series in labels order, otherwise // merged series set will be incorrect. -// Argument 'querier' is optional and can be nil. Pass Querier if you want to retry query in case of failing series set. -// Overlapped situations are merged using provided mergeFunc. -func newGenericMergeSeriesSet(sets []genericSeriesSet, querier *mergeGenericQuerier, mergeFunc genericSeriesMergeFunc) genericSeriesSet { +// Overlapping cases are merged using provided mergeFn. +func newGenericMergeSeriesSet(sets []genericSeriesSet, mergeFn genericSeriesMergeFunc) genericSeriesSet { if len(sets) == 1 { return sets[0] } - // Sets need to be pre-advanced, so we can introspect the label of the + // We are pre-advancing sets, so we can introspect the label of the // series under the cursor. var h genericSeriesSetHeap for _, set := range sets { @@ -483,26 +459,25 @@ func newGenericMergeSeriesSet(sets []genericSeriesSet, querier *mergeGenericQuer } } return &genericMergeSeriesSet{ - mergeFunc: mergeFunc, - heap: h, - sets: sets, - querier: querier, + mergeFn: mergeFn, + sets: sets, + heap: h, } } func (c *genericMergeSeriesSet) Next() bool { // Run in a loop because the "next" series sets may not be valid anymore. - // If a remote querier fails, we discard all series sets from that querier. // If, for the current label set, all the next series sets come from // failed remote storage sources, we want to keep trying with the next label set. for { - // Firstly advance all the current series sets. If any of them have run out + // Firstly advance all the current series sets. If any of them have run out // we can drop them, otherwise they should be inserted back into the heap. for _, set := range c.currentSets { if set.Next() { heap.Push(&c.heap, set) } } + if len(c.heap) == 0 { return false } @@ -512,9 +487,6 @@ func (c *genericMergeSeriesSet) Next() bool { c.currentLabels = c.heap[0].At().Labels() for len(c.heap) > 0 && labels.Equal(c.currentLabels, c.heap[0].At().Labels()) { set := heap.Pop(&c.heap).(genericSeriesSet) - if c.querier != nil && c.querier.IsFailedSet(set) { - continue - } c.currentSets = append(c.currentSets, set) } @@ -535,7 +507,7 @@ func (c *genericMergeSeriesSet) At() Labels { for _, seriesSet := range c.currentSets { series = append(series, seriesSet.At()) } - return c.mergeFunc(series...) + return c.mergeFn(series...) } func (c *genericMergeSeriesSet) Err() error { @@ -547,6 +519,14 @@ func (c *genericMergeSeriesSet) Err() error { return nil } +func (c *genericMergeSeriesSet) Warnings() Warnings { + var ws Warnings + for _, set := range c.sets { + ws = append(ws, set.Warnings()...) + } + return ws +} + type genericSeriesSetHeap []genericSeriesSet func (h genericSeriesSetHeap) Len() int { return len(h) } diff --git a/vendor/github.com/prometheus/prometheus/storage/generic.go b/vendor/github.com/prometheus/prometheus/storage/generic.go index d23e51a6c6188..36283a40a89d4 100644 --- a/vendor/github.com/prometheus/prometheus/storage/generic.go +++ b/vendor/github.com/prometheus/prometheus/storage/generic.go @@ -20,13 +20,14 @@ import "github.com/prometheus/prometheus/pkg/labels" type genericQuerier interface { baseQuerier - Select(bool, *SelectHints, ...*labels.Matcher) (genericSeriesSet, Warnings, error) + Select(bool, *SelectHints, ...*labels.Matcher) genericSeriesSet } type genericSeriesSet interface { Next() bool At() Labels Err() error + Warnings() Warnings } type genericSeriesMergeFunc func(...Labels) Labels @@ -55,13 +56,11 @@ type genericQuerierAdapter struct { cq ChunkQuerier } -func (q *genericQuerierAdapter) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) (genericSeriesSet, Warnings, error) { +func (q *genericQuerierAdapter) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) genericSeriesSet { if q.q != nil { - s, w, err := q.q.Select(sortSeries, hints, matchers...) - return &genericSeriesSetAdapter{s}, w, err + return &genericSeriesSetAdapter{q.q.Select(sortSeries, hints, matchers...)} } - s, w, err := q.cq.Select(sortSeries, hints, matchers...) - return &genericChunkSeriesSetAdapter{s}, w, err + return &genericChunkSeriesSetAdapter{q.cq.Select(sortSeries, hints, matchers...)} } func newGenericQuerierFrom(q Querier) genericQuerier { @@ -84,9 +83,8 @@ func (a *seriesSetAdapter) At() Series { return a.genericSeriesSet.At().(Series) } -func (q *querierAdapter) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) (SeriesSet, Warnings, error) { - s, w, err := q.genericQuerier.Select(sortSeries, hints, matchers...) - return &seriesSetAdapter{s}, w, err +func (q *querierAdapter) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) SeriesSet { + return &seriesSetAdapter{q.genericQuerier.Select(sortSeries, hints, matchers...)} } type chunkQuerierAdapter struct { @@ -101,9 +99,8 @@ func (a *chunkSeriesSetAdapter) At() ChunkSeries { return a.genericSeriesSet.At().(ChunkSeries) } -func (q *chunkQuerierAdapter) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) (ChunkSeriesSet, Warnings, error) { - s, w, err := q.genericQuerier.Select(sortSeries, hints, matchers...) - return &chunkSeriesSetAdapter{s}, w, err +func (q *chunkQuerierAdapter) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) ChunkSeriesSet { + return &chunkSeriesSetAdapter{q.genericQuerier.Select(sortSeries, hints, matchers...)} } type seriesMergerAdapter struct { @@ -129,3 +126,13 @@ func (a *chunkSeriesMergerAdapter) Merge(s ...Labels) Labels { } return a.VerticalChunkSeriesMergerFunc(buf...) } + +type noopGenericSeriesSet struct{} + +func (noopGenericSeriesSet) Next() bool { return false } + +func (noopGenericSeriesSet) At() Labels { return nil } + +func (noopGenericSeriesSet) Err() error { return nil } + +func (noopGenericSeriesSet) Warnings() Warnings { return nil } diff --git a/vendor/github.com/prometheus/prometheus/storage/interface.go b/vendor/github.com/prometheus/prometheus/storage/interface.go index 0dc3b9efb2e68..dd558bb5381c2 100644 --- a/vendor/github.com/prometheus/prometheus/storage/interface.go +++ b/vendor/github.com/prometheus/prometheus/storage/interface.go @@ -65,7 +65,7 @@ type Querier interface { // Select returns a set of series that matches the given label matchers. // Caller can specify if it requires returned series to be sorted. Prefer not requiring sorting for better performance. // It allows passing hints that can help in optimising select, but it's up to implementation how this is used if used at all. - Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) (SeriesSet, Warnings, error) + Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) SeriesSet } // A ChunkQueryable handles queries against a storage. @@ -82,7 +82,7 @@ type ChunkQuerier interface { // Select returns a set of series that matches the given label matchers. // Caller can specify if it requires returned series to be sorted. Prefer not requiring sorting for better performance. // It allows passing hints that can help in optimising select, but it's up to implementation how this is used if used at all. - Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) (ChunkSeriesSet, Warnings, error) + Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) ChunkSeriesSet } type baseQuerier interface { @@ -153,7 +153,12 @@ type Appender interface { type SeriesSet interface { Next() bool At() Series + // The error that iteration as failed with. + // When an error occurs, set cannot continue to iterate. Err() error + // A collection of warnings for the whole set. + // Warnings could be return even iteration has not failed with error. + Warnings() Warnings } var emptySeriesSet = errSeriesSet{} @@ -164,12 +169,19 @@ func EmptySeriesSet() SeriesSet { } type errSeriesSet struct { + ws Warnings err error } -func (s errSeriesSet) Next() bool { return false } -func (s errSeriesSet) At() Series { return nil } -func (s errSeriesSet) Err() error { return s.err } +func (s errSeriesSet) Next() bool { return false } +func (s errSeriesSet) At() Series { return nil } +func (s errSeriesSet) Err() error { return s.err } +func (s errSeriesSet) Warnings() Warnings { return s.ws } + +// ErrSeriesSet returns a series set that wraps an error. +func ErrSeriesSet(err error) SeriesSet { + return errSeriesSet{err: err} +} // Series exposes a single time series and allows iterating over samples. type Series interface { @@ -181,7 +193,12 @@ type Series interface { type ChunkSeriesSet interface { Next() bool At() ChunkSeries + // The error that iteration has failed with. + // When an error occurs, set cannot continue to iterate. Err() error + // A collection of warnings for the whole set. + // Warnings could be return even iteration has not failed with error. + Warnings() Warnings } // ChunkSeries exposes a single time series and allows iterating over chunks. diff --git a/vendor/github.com/prometheus/prometheus/storage/lazy.go b/vendor/github.com/prometheus/prometheus/storage/lazy.go new file mode 100644 index 0000000000000..8eaa8772abbbb --- /dev/null +++ b/vendor/github.com/prometheus/prometheus/storage/lazy.go @@ -0,0 +1,67 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +type lazySeriesSet struct { + create func() (s genericSeriesSet, ok bool) + + set genericSeriesSet +} + +func (c *lazySeriesSet) Next() bool { + if c.set != nil { + return c.set.Next() + } + + var ok bool + c.set, ok = c.create() + return ok +} + +func (c *lazySeriesSet) Err() error { + if c.set != nil { + return c.set.Err() + } + return nil +} + +func (c *lazySeriesSet) At() Labels { + if c.set != nil { + return c.set.At() + } + return nil +} + +func (c *lazySeriesSet) Warnings() Warnings { + if c.set != nil { + return c.set.Warnings() + } + return nil +} + +type warningsOnlySeriesSet Warnings + +func (warningsOnlySeriesSet) Next() bool { return false } +func (warningsOnlySeriesSet) Err() error { return nil } +func (warningsOnlySeriesSet) At() Labels { return nil } +func (c warningsOnlySeriesSet) Warnings() Warnings { return Warnings(c) } + +type errorOnlySeriesSet struct { + err error +} + +func (errorOnlySeriesSet) Next() bool { return false } +func (errorOnlySeriesSet) At() Labels { return nil } +func (s errorOnlySeriesSet) Err() error { return s.err } +func (errorOnlySeriesSet) Warnings() Warnings { return nil } diff --git a/vendor/github.com/prometheus/prometheus/storage/noop.go b/vendor/github.com/prometheus/prometheus/storage/noop.go index c4ab1ba340ff7..00599aba748a3 100644 --- a/vendor/github.com/prometheus/prometheus/storage/noop.go +++ b/vendor/github.com/prometheus/prometheus/storage/noop.go @@ -24,8 +24,8 @@ func NoopQuerier() Querier { return noopQuerier{} } -func (noopQuerier) Select(bool, *SelectHints, ...*labels.Matcher) (SeriesSet, Warnings, error) { - return NoopSeriesSet(), nil, nil +func (noopQuerier) Select(bool, *SelectHints, ...*labels.Matcher) SeriesSet { + return NoopSeriesSet() } func (noopQuerier) LabelValues(string) ([]string, Warnings, error) { @@ -47,8 +47,8 @@ func NoopChunkedQuerier() ChunkQuerier { return noopChunkQuerier{} } -func (noopChunkQuerier) Select(bool, *SelectHints, ...*labels.Matcher) (ChunkSeriesSet, Warnings, error) { - return NoopChunkedSeriesSet(), nil, nil +func (noopChunkQuerier) Select(bool, *SelectHints, ...*labels.Matcher) ChunkSeriesSet { + return NoopChunkedSeriesSet() } func (noopChunkQuerier) LabelValues(string) ([]string, Warnings, error) { @@ -76,6 +76,8 @@ func (noopSeriesSet) At() Series { return nil } func (noopSeriesSet) Err() error { return nil } +func (noopSeriesSet) Warnings() Warnings { return nil } + type noopChunkedSeriesSet struct{} // NoopChunkedSeriesSet is a ChunkSeriesSet that does nothing. @@ -88,3 +90,5 @@ func (noopChunkedSeriesSet) Next() bool { return false } func (noopChunkedSeriesSet) At() ChunkSeries { return nil } func (noopChunkedSeriesSet) Err() error { return nil } + +func (noopChunkedSeriesSet) Warnings() Warnings { return nil } diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/client.go b/vendor/github.com/prometheus/prometheus/storage/remote/client.go index 4dce3ec50e204..7ff957168c3af 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/client.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/client.go @@ -21,6 +21,7 @@ import ( "io" "io/ioutil" "net/http" + "strconv" "strings" "time" @@ -28,6 +29,7 @@ import ( "github.com/golang/snappy" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" config_util "github.com/prometheus/common/config" "github.com/prometheus/common/model" "github.com/prometheus/common/version" @@ -40,6 +42,16 @@ const maxErrMsgLen = 256 var userAgent = fmt.Sprintf("Prometheus/%s", version.Version) +var remoteReadQueriesTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "read_queries_total", + Help: "The total number of remote read queries.", + }, + []string{remoteName, endpoint, "code"}, +) + // Client allows reading and writing from/to a remote HTTP endpoint. type Client struct { remoteName string // Used to differentiate clients in metrics. @@ -55,6 +67,10 @@ type ClientConfig struct { HTTPClientConfig config_util.HTTPClientConfig } +func init() { + prometheus.MustRegister(remoteReadQueriesTotal) +} + // NewClient creates a new Client. func NewClient(remoteName string, conf *ClientConfig) (*Client, error) { httpClient, err := config_util.NewClientFromConfig(conf.HTTPClientConfig, "remote_storage", false) @@ -193,6 +209,9 @@ func (c *Client) Read(ctx context.Context, query *prompb.Query) (*prompb.QueryRe httpResp.Body.Close() }() + remoteReadTotalCounter := remoteReadQueriesTotal.WithLabelValues(c.remoteName, c.url.String(), strconv.Itoa(httpResp.StatusCode)) + remoteReadTotalCounter.Inc() + compressed, err = ioutil.ReadAll(httpResp.Body) if err != nil { return nil, errors.Wrap(err, fmt.Sprintf("error reading response. HTTP status code: %s", httpResp.Status)) diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/codec.go b/vendor/github.com/prometheus/prometheus/storage/remote/codec.go index 3b6556a1241e5..6985a5f6a6674 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/codec.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/codec.go @@ -107,7 +107,7 @@ func ToQuery(from, to int64, matchers []*labels.Matcher, hints *storage.SelectHi } // ToQueryResult builds a QueryResult proto. -func ToQueryResult(ss storage.SeriesSet, sampleLimit int) (*prompb.QueryResult, error) { +func ToQueryResult(ss storage.SeriesSet, sampleLimit int) (*prompb.QueryResult, storage.Warnings, error) { numSamples := 0 resp := &prompb.QueryResult{} for ss.Next() { @@ -118,7 +118,7 @@ func ToQueryResult(ss storage.SeriesSet, sampleLimit int) (*prompb.QueryResult, for iter.Next() { numSamples++ if sampleLimit > 0 && numSamples > sampleLimit { - return nil, HTTPError{ + return nil, ss.Warnings(), HTTPError{ msg: fmt.Sprintf("exceeded sample limit (%d)", sampleLimit), status: http.StatusBadRequest, } @@ -130,7 +130,7 @@ func ToQueryResult(ss storage.SeriesSet, sampleLimit int) (*prompb.QueryResult, }) } if err := iter.Err(); err != nil { - return nil, err + return nil, ss.Warnings(), err } resp.Timeseries = append(resp.Timeseries, &prompb.TimeSeries{ @@ -139,9 +139,9 @@ func ToQueryResult(ss storage.SeriesSet, sampleLimit int) (*prompb.QueryResult, }) } if err := ss.Err(); err != nil { - return nil, err + return nil, ss.Warnings(), err } - return resp, nil + return resp, ss.Warnings(), nil } // FromQueryResult unpacks and sorts a QueryResult proto. @@ -195,7 +195,7 @@ func StreamChunkedReadResponses( ss storage.SeriesSet, sortedExternalLabels []prompb.Label, maxBytesInFrame int, -) error { +) (storage.Warnings, error) { var ( chks []prompb.Chunk lbls []prompb.Label @@ -218,7 +218,7 @@ func StreamChunkedReadResponses( // TODO(bwplotka): Use ChunkIterator once available in TSDB instead of re-encoding: https://github.com/prometheus/prometheus/pull/5882 chks, err = encodeChunks(iter, chks, maxBytesInFrame-lblsSize) if err != nil { - return err + return ss.Warnings(), err } if len(chks) == 0 { @@ -234,25 +234,25 @@ func StreamChunkedReadResponses( QueryIndex: queryIndex, }) if err != nil { - return errors.Wrap(err, "marshal ChunkedReadResponse") + return ss.Warnings(), errors.Wrap(err, "marshal ChunkedReadResponse") } if _, err := stream.Write(b); err != nil { - return errors.Wrap(err, "write to stream") + return ss.Warnings(), errors.Wrap(err, "write to stream") } chks = chks[:0] } if err := iter.Err(); err != nil { - return err + return ss.Warnings(), err } } if err := ss.Err(); err != nil { - return err + return ss.Warnings(), err } - return nil + return ss.Warnings(), nil } // encodeChunks expects iterator to be ready to use (aka iter.Next() called before invoking). @@ -365,6 +365,8 @@ func (e errSeriesSet) Err() error { return e.err } +func (e errSeriesSet) Warnings() storage.Warnings { return nil } + // concreteSeriesSet implements storage.SeriesSet. type concreteSeriesSet struct { cur int @@ -384,6 +386,8 @@ func (c *concreteSeriesSet) Err() error { return nil } +func (c *concreteSeriesSet) Warnings() storage.Warnings { return nil } + // concreteSeries implements storage.Series. type concreteSeries struct { labels labels.Labels diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/queue_manager.go b/vendor/github.com/prometheus/prometheus/storage/remote/queue_manager.go index 6a8b4c4325c66..befa9a7b7b0ba 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/queue_manager.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/queue_manager.go @@ -115,7 +115,7 @@ func newQueueManagerMetrics(r prometheus.Registerer, rn, e string) *queueManager Subsystem: subsystem, Name: "sent_batch_duration_seconds", Help: "Duration of sample batch send calls to the remote storage.", - Buckets: prometheus.DefBuckets, + Buckets: append(prometheus.DefBuckets, 25, 60, 120, 300), ConstLabels: constLabels, }) m.highestSentTimestamp = &maxGauge{ diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/read.go b/vendor/github.com/prometheus/prometheus/storage/remote/read.go index 219ba3ab3844c..0b53ed7e74a43 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/read.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/read.go @@ -16,6 +16,7 @@ package remote import ( "context" "fmt" + "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/pkg/labels" @@ -32,26 +33,26 @@ var remoteReadQueries = prometheus.NewGaugeVec( []string{remoteName, endpoint}, ) -var remoteReadQueriesTotal = prometheus.NewCounterVec( - prometheus.CounterOpts{ +var remoteReadQueriesHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ Namespace: namespace, Subsystem: subsystem, - Name: "remote_read_queries_total", - Help: "The total number of remote read queries.", + Name: "read_request_duration_seconds", + Help: "Histogram of the latency for remote read requests.", + Buckets: append(prometheus.DefBuckets, 25, 60), }, []string{remoteName, endpoint}, ) func init() { - prometheus.MustRegister(remoteReadQueries) - prometheus.MustRegister(remoteReadQueriesTotal) + prometheus.MustRegister(remoteReadQueries, remoteReadQueriesHistogram) } // QueryableClient returns a storage.Queryable which queries the given // Client to select series sets. func QueryableClient(c *Client) storage.Queryable { remoteReadQueries.WithLabelValues(c.remoteName, c.url.String()) - remoteReadQueriesTotal.WithLabelValues(c.remoteName, c.url.String()) + remoteReadQueriesHistogram.WithLabelValues(c.remoteName, c.url.String()) return storage.QueryableFunc(func(ctx context.Context, mint, maxt int64) (storage.Querier, error) { return &querier{ @@ -71,25 +72,25 @@ type querier struct { } // Select implements storage.Querier and uses the given matchers to read series sets from the Client. -func (q *querier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (q *querier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { query, err := ToQuery(q.mint, q.maxt, matchers, hints) if err != nil { - return nil, nil, err + return storage.ErrSeriesSet(err) } remoteReadGauge := remoteReadQueries.WithLabelValues(q.client.remoteName, q.client.url.String()) remoteReadGauge.Inc() defer remoteReadGauge.Dec() - remoteReadTotalCounter := remoteReadQueriesTotal.WithLabelValues(q.client.remoteName, q.client.url.String()) - remoteReadTotalCounter.Inc() + remoteReadQueriesHistogram := remoteReadQueriesHistogram.WithLabelValues(q.client.remoteName, q.client.url.String()) + remoteReadQueriesHistogram.Observe(time.Since(time.Now()).Seconds()) res, err := q.client.Read(q.ctx, query) if err != nil { - return nil, nil, fmt.Errorf("remote_read: %v", err) + return storage.ErrSeriesSet(fmt.Errorf("remote_read: %v", err)) } - return FromQueryResult(sortSeries, res), nil, nil + return FromQueryResult(sortSeries, res) } // LabelValues implements storage.Querier and is a noop. @@ -132,13 +133,9 @@ type externalLabelsQuerier struct { // Select adds equality matchers for all external labels to the list of matchers // before calling the wrapped storage.Queryable. The added external labels are // removed from the returned series sets. -func (q externalLabelsQuerier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (q externalLabelsQuerier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { m, added := q.addExternalLabels(matchers) - s, warnings, err := q.Querier.Select(sortSeries, hints, m...) - if err != nil { - return nil, warnings, err - } - return newSeriesSetFilter(s, added), warnings, nil + return newSeriesSetFilter(q.Querier.Select(sortSeries, hints, m...), added) } // PreferLocalStorageFilter returns a QueryableFunc which creates a NoopQuerier @@ -185,7 +182,7 @@ type requiredMatchersQuerier struct { // Select returns a NoopSeriesSet if the given matchers don't match the label // set of the requiredMatchersQuerier. Otherwise it'll call the wrapped querier. -func (q requiredMatchersQuerier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (q requiredMatchersQuerier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { ms := q.requiredMatchers for _, m := range matchers { for i, r := range ms { @@ -199,7 +196,7 @@ func (q requiredMatchersQuerier) Select(sortSeries bool, hints *storage.SelectHi } } if len(ms) > 0 { - return storage.NoopSeriesSet(), nil, nil + return storage.NoopSeriesSet() } return q.Querier.Select(sortSeries, hints, matchers...) } diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/storage.go b/vendor/github.com/prometheus/prometheus/storage/remote/storage.go index 1b756af11234b..9f2d8a58d86e6 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/storage.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/storage.go @@ -132,6 +132,9 @@ func (s *Storage) StartTime() (int64, error) { // Querier returns a storage.MergeQuerier combining the remote client queriers // of each configured remote read endpoint. +// Returned querier will never return error as all queryables are assumed best effort. +// Additionally all returned queriers ensure that its Select's SeriesSets have ready data after first `Next` invoke. +// This is because Prometheus (fanout and secondary queries) can't handle the stream failing half way through by design. func (s *Storage) Querier(ctx context.Context, mint, maxt int64) (storage.Querier, error) { s.mtx.Lock() queryables := s.queryables diff --git a/vendor/github.com/prometheus/prometheus/storage/secondary.go b/vendor/github.com/prometheus/prometheus/storage/secondary.go new file mode 100644 index 0000000000000..7f70930697f06 --- /dev/null +++ b/vendor/github.com/prometheus/prometheus/storage/secondary.go @@ -0,0 +1,112 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "sync" + + "github.com/prometheus/prometheus/pkg/labels" +) + +// secondaryQuerier is a wrapper that allows a querier to be treated in a best effort manner. +// This means that an error on any method returned by Querier except Close will be returned as a warning, +// and the result will be empty. +// +// Additionally, Querier ensures that if ANY SeriesSet returned by this querier's Select failed on an initial Next, +// All other SeriesSet will be return no response as well. This ensures consistent partial response strategy, where you +// have either full results or none from each secondary Querier. +// NOTE: This works well only for implementations that only fail during first Next() (e.g fetch from network). If implementation fails +// during further iterations, set will panic. If Select is invoked after first Next of any returned SeriesSet, querier will panic. +// +// Not go-routine safe. +// NOTE: Prometheus treats all remote storages as secondary / best effort. +type secondaryQuerier struct { + genericQuerier + + once sync.Once + done bool + asyncSets []genericSeriesSet +} + +func newSecondaryQuerierFrom(q Querier) genericQuerier { + return &secondaryQuerier{genericQuerier: newGenericQuerierFrom(q)} +} + +func newSecondaryQuerierFromChunk(cq ChunkQuerier) genericQuerier { + return &secondaryQuerier{genericQuerier: newGenericQuerierFromChunk(cq)} +} + +func (s *secondaryQuerier) LabelValues(name string) ([]string, Warnings, error) { + vals, w, err := s.genericQuerier.LabelValues(name) + if err != nil { + return nil, append([]error{err}, w...), nil + } + return vals, w, nil +} + +func (s *secondaryQuerier) LabelNames() ([]string, Warnings, error) { + names, w, err := s.genericQuerier.LabelNames() + if err != nil { + return nil, append([]error{err}, w...), nil + } + return names, w, nil +} + +func (s *secondaryQuerier) createFn(asyncSet genericSeriesSet) func() (genericSeriesSet, bool) { + s.asyncSets = append(s.asyncSets, asyncSet) + curr := len(s.asyncSets) - 1 + return func() (genericSeriesSet, bool) { + s.once.Do(func() { + // At first create invocation we iterate over all sets and ensure its Next() returns some value without + // errors. This is to ensure we support consistent partial failures. + for i, set := range s.asyncSets { + if set.Next() { + continue + } + ws := set.Warnings() + // Failed set. + if err := set.Err(); err != nil { + ws = append([]error{err}, ws...) + // Promote the warnings to the current one. + s.asyncSets[curr] = warningsOnlySeriesSet(ws) + // One of the sets failed, ensure rest of the sets returns nothing. (All or nothing logic). + for i := range s.asyncSets { + if curr != i { + s.asyncSets[i] = noopGenericSeriesSet{} + } + } + break + } + // Exhausted set. + s.asyncSets[i] = warningsOnlySeriesSet(ws) + } + + s.done = true + }) + + switch s.asyncSets[curr].(type) { + case warningsOnlySeriesSet, noopGenericSeriesSet: + return s.asyncSets[curr], false + default: + return s.asyncSets[curr], true + } + } +} + +func (s *secondaryQuerier) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) genericSeriesSet { + if s.done { + panic("secondaryQuerier: Select invoked after first Next of any returned SeriesSet was done") + } + return &lazySeriesSet{create: s.createFn(s.genericQuerier.Select(sortSeries, hints, matchers...))} +} diff --git a/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/bstream.go b/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/bstream.go index 0a02a730358ca..ad8077c27e9f3 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/bstream.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/bstream.go @@ -41,7 +41,10 @@ package chunkenc -import "io" +import ( + "encoding/binary" + "io" +) // bstream is a stream of bits. type bstream struct { @@ -49,10 +52,6 @@ type bstream struct { count uint8 // how many bits are valid in current byte } -func newBReader(b []byte) bstream { - return bstream{stream: b, count: 8} -} - func (b *bstream) bytes() []byte { return b.stream } @@ -111,90 +110,131 @@ func (b *bstream) writeBits(u uint64, nbits int) { } } -func (b *bstream) readBit() (bit, error) { - if len(b.stream) == 0 { - return false, io.EOF - } +type bstreamReader struct { + stream []byte + streamOffset int // The offset from which read the next byte from the stream. - if b.count == 0 { - b.stream = b.stream[1:] + buffer uint64 // The current buffer, filled from the stream, containing up to 8 bytes from which read bits. + valid uint8 // The number of bits valid to read (from left) in the current buffer. +} - if len(b.stream) == 0 { +func newBReader(b []byte) bstreamReader { + return bstreamReader{ + stream: b, + } +} + +func (b *bstreamReader) readBit() (bit, error) { + if b.valid == 0 { + if !b.loadNextBuffer(1) { return false, io.EOF } - b.count = 8 } - d := (b.stream[0] << (8 - b.count)) & 0x80 - b.count-- - return d != 0, nil -} - -func (b *bstream) ReadByte() (byte, error) { - return b.readByte() + return b.readBitFast() } -func (b *bstream) readByte() (byte, error) { - if len(b.stream) == 0 { - return 0, io.EOF +// readBitFast is like readBit but can return io.EOF if the internal buffer is empty. +// If it returns io.EOF, the caller should retry reading bits calling readBit(). +// This function must be kept small and a leaf in order to help the compiler inlining it +// and further improve performances. +func (b *bstreamReader) readBitFast() (bit, error) { + if b.valid == 0 { + return false, io.EOF } - if b.count == 0 { - b.stream = b.stream[1:] + b.valid-- + bitmask := uint64(1) << b.valid + return (b.buffer & bitmask) != 0, nil +} - if len(b.stream) == 0 { +func (b *bstreamReader) readBits(nbits uint8) (uint64, error) { + if b.valid == 0 { + if !b.loadNextBuffer(nbits) { return 0, io.EOF } - return b.stream[0], nil } - if b.count == 8 { - b.count = 0 - return b.stream[0], nil + if nbits <= b.valid { + return b.readBitsFast(nbits) } - byt := b.stream[0] << (8 - b.count) - b.stream = b.stream[1:] + // We have to read all remaining valid bits from the current buffer and a part from the next one. + bitmask := (uint64(1) << b.valid) - 1 + nbits -= b.valid + v := (b.buffer & bitmask) << nbits + b.valid = 0 - if len(b.stream) == 0 { + if !b.loadNextBuffer(nbits) { return 0, io.EOF } - // We just advanced the stream and can assume the shift to be 0. - byt |= b.stream[0] >> b.count + bitmask = (uint64(1) << nbits) - 1 + v = v | ((b.buffer >> (b.valid - nbits)) & bitmask) + b.valid -= nbits - return byt, nil + return v, nil } -func (b *bstream) readBits(nbits int) (uint64, error) { - var u uint64 +// readBitsFast is like readBits but can return io.EOF if the internal buffer is empty. +// If it returns io.EOF, the caller should retry reading bits calling readBits(). +// This function must be kept small and a leaf in order to help the compiler inlining it +// and further improve performances. +func (b *bstreamReader) readBitsFast(nbits uint8) (uint64, error) { + if nbits > b.valid { + return 0, io.EOF + } - for nbits >= 8 { - byt, err := b.readByte() - if err != nil { - return 0, err - } + bitmask := (uint64(1) << nbits) - 1 + b.valid -= nbits - u = (u << 8) | uint64(byt) - nbits -= 8 + return (b.buffer >> b.valid) & bitmask, nil +} + +func (b *bstreamReader) ReadByte() (byte, error) { + v, err := b.readBits(8) + if err != nil { + return 0, err } + return byte(v), nil +} - if nbits == 0 { - return u, nil +// loadNextBuffer loads the next bytes from the stream into the internal buffer. +// The input nbits is the minimum number of bits that must be read, but the implementation +// can read more (if possible) to improve performances. +func (b *bstreamReader) loadNextBuffer(nbits uint8) bool { + if b.streamOffset >= len(b.stream) { + return false } - if nbits > int(b.count) { - u = (u << uint(b.count)) | uint64((b.stream[0]<<(8-b.count))>>(8-b.count)) - nbits -= int(b.count) - b.stream = b.stream[1:] + // Handle the case there are more then 8 bytes in the buffer (most common case) + // in a optimized way. It's guaranteed that this branch will never read from the + // very last byte of the stream (which suffers race conditions due to concurrent + // writes). + if b.streamOffset+8 < len(b.stream) { + b.buffer = binary.BigEndian.Uint64(b.stream[b.streamOffset:]) + b.streamOffset += 8 + b.valid = 64 + return true + } - if len(b.stream) == 0 { - return 0, io.EOF - } - b.count = 8 + // We're here if the are 8 or less bytes left in the stream. Since this reader needs + // to handle race conditions with concurrent writes happening on the very last byte + // we make sure to never over more than the minimum requested bits (rounded up to + // the next byte). The following code is slower but called less frequently. + nbytes := int((nbits / 8) + 1) + if b.streamOffset+nbytes > len(b.stream) { + nbytes = len(b.stream) - b.streamOffset + } + + buffer := uint64(0) + for i := 0; i < nbytes; i++ { + buffer = buffer | (uint64(b.stream[b.streamOffset+i]) << uint(8*(nbytes-i-1))) } - u = (u << uint(nbits)) | uint64((b.stream[0]<<(8-b.count))>>(8-uint(nbits))) - b.count -= uint8(nbits) - return u, nil + b.buffer = buffer + b.streamOffset += nbytes + b.valid = uint8(nbytes * 8) + + return true } diff --git a/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/xor.go b/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/xor.go index 33c7f104042d9..e3b4f58b2ad06 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/xor.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/xor.go @@ -240,7 +240,7 @@ func (a *xorAppender) writeVDelta(v float64) { } type xorIterator struct { - br bstream + br bstreamReader numTotal uint16 numRead uint16 @@ -328,7 +328,10 @@ func (it *xorIterator) Next() bool { // read delta-of-delta for i := 0; i < 4; i++ { d <<= 1 - bit, err := it.br.readBit() + bit, err := it.br.readBitFast() + if err != nil { + bit, err = it.br.readBit() + } if err != nil { it.err = err return false @@ -350,6 +353,7 @@ func (it *xorIterator) Next() bool { case 0x0e: sz = 20 case 0x0f: + // Do not use fast because it's very unlikely it will succeed. bits, err := it.br.readBits(64) if err != nil { it.err = err @@ -360,7 +364,10 @@ func (it *xorIterator) Next() bool { } if sz != 0 { - bits, err := it.br.readBits(int(sz)) + bits, err := it.br.readBitsFast(sz) + if err != nil { + bits, err = it.br.readBits(sz) + } if err != nil { it.err = err return false @@ -379,7 +386,10 @@ func (it *xorIterator) Next() bool { } func (it *xorIterator) readValue() bool { - bit, err := it.br.readBit() + bit, err := it.br.readBitFast() + if err != nil { + bit, err = it.br.readBit() + } if err != nil { it.err = err return false @@ -388,7 +398,10 @@ func (it *xorIterator) readValue() bool { if bit == zero { // it.val = it.val } else { - bit, err := it.br.readBit() + bit, err := it.br.readBitFast() + if err != nil { + bit, err = it.br.readBit() + } if err != nil { it.err = err return false @@ -397,14 +410,20 @@ func (it *xorIterator) readValue() bool { // reuse leading/trailing zero bits // it.leading, it.trailing = it.leading, it.trailing } else { - bits, err := it.br.readBits(5) + bits, err := it.br.readBitsFast(5) + if err != nil { + bits, err = it.br.readBits(5) + } if err != nil { it.err = err return false } it.leading = uint8(bits) - bits, err = it.br.readBits(6) + bits, err = it.br.readBitsFast(6) + if err != nil { + bits, err = it.br.readBits(6) + } if err != nil { it.err = err return false @@ -417,14 +436,17 @@ func (it *xorIterator) readValue() bool { it.trailing = 64 - it.leading - mbits } - mbits := int(64 - it.leading - it.trailing) - bits, err := it.br.readBits(mbits) + mbits := 64 - it.leading - it.trailing + bits, err := it.br.readBitsFast(mbits) + if err != nil { + bits, err = it.br.readBits(mbits) + } if err != nil { it.err = err return false } vbits := math.Float64bits(it.val) - vbits ^= (bits << it.trailing) + vbits ^= bits << it.trailing it.val = math.Float64frombits(vbits) } diff --git a/vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks.go b/vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks.go index a7edbe982cce6..40b14afcdf01b 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks.go @@ -190,23 +190,23 @@ func (cdm *ChunkDiskMapper) openMMapFiles() (returnErr error) { lastSeq := chkFileIndices[0] for _, seq := range chkFileIndices[1:] { if seq != lastSeq+1 { - return errors.Errorf("found unsequential head chunk files %d and %d", lastSeq, seq) + return errors.Errorf("found unsequential head chunk files %s (index: %d) and %s (index: %d)", files[lastSeq], lastSeq, files[seq], seq) } lastSeq = seq } for i, b := range cdm.mmappedChunkFiles { if b.byteSlice.Len() < HeadChunkFileHeaderSize { - return errors.Wrapf(errInvalidSize, "invalid head chunk file header in file %d", i) + return errors.Wrapf(errInvalidSize, "%s: invalid head chunk file header", files[i]) } // Verify magic number. if m := binary.BigEndian.Uint32(b.byteSlice.Range(0, MagicChunksSize)); m != MagicHeadChunks { - return errors.Errorf("invalid magic number %x", m) + return errors.Errorf("%s: invalid magic number %x", files[i], m) } // Verify chunk format version. if v := int(b.byteSlice.Range(MagicChunksSize, MagicChunksSize+ChunksFormatVersionSize)[0]); v != chunksFormatV1 { - return errors.Errorf("invalid chunk format version %d", v) + return errors.Errorf("%s: invalid chunk format version %d", files[i], v) } cdm.size += int64(b.byteSlice.Len()) @@ -670,8 +670,8 @@ func (cdm *ChunkDiskMapper) Truncate(mint int64) error { var removedFiles []int for _, seq := range chkFileIndices { - if seq == cdm.curFileSequence { - continue + if seq == cdm.curFileSequence || cdm.mmappedChunkFiles[seq].maxt >= mint { + break } if cdm.mmappedChunkFiles[seq].maxt < mint { removedFiles = append(removedFiles, seq) @@ -680,7 +680,10 @@ func (cdm *ChunkDiskMapper) Truncate(mint int64) error { cdm.readPathMtx.RUnlock() var merr tsdb_errors.MultiError - merr.Add(cdm.CutNewFile()) + // Cut a new file only if the current file has some chunks. + if cdm.curFileNumBytes > HeadChunkFileHeaderSize { + merr.Add(cdm.CutNewFile()) + } merr.Add(cdm.deleteFiles(removedFiles)) return merr.Err() } diff --git a/vendor/github.com/prometheus/prometheus/tsdb/db.go b/vendor/github.com/prometheus/prometheus/tsdb/db.go index 1997dca2d6420..a2892b6d9e8bc 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/db.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/db.go @@ -441,10 +441,14 @@ func (db *DBReadOnly) Blocks() ([]BlockReader, error) { if len(corrupted) > 0 { for _, b := range loadable { if err := b.Close(); err != nil { - level.Warn(db.logger).Log("msg", "Closing a block", err) + level.Warn(db.logger).Log("msg", "Closing block failed", "err", err, "block", b) } } - return nil, errors.Errorf("unexpected corrupted block:%v", corrupted) + var merr tsdb_errors.MultiError + for ulid, err := range corrupted { + merr.Add(errors.Wrapf(err, "corrupted block %s", ulid.String())) + } + return nil, merr.Err() } if len(loadable) == 0 { @@ -880,7 +884,11 @@ func (db *DB) reload() (err error) { block.Close() } } - return fmt.Errorf("unexpected corrupted block:%v", corrupted) + var merr tsdb_errors.MultiError + for ulid, err := range corrupted { + merr.Add(errors.Wrapf(err, "corrupted block %s", ulid.String())) + } + return merr.Err() } // All deletable blocks should not be loaded. @@ -1054,7 +1062,7 @@ func (db *DB) deleteBlocks(blocks map[ulid.ULID]*Block) error { for ulid, block := range blocks { if block != nil { if err := block.Close(); err != nil { - level.Warn(db.logger).Log("msg", "Closing block failed", "err", err) + level.Warn(db.logger).Log("msg", "Closing block failed", "err", err, "block", ulid) } } if err := os.RemoveAll(filepath.Join(db.dir, ulid.String())); err != nil { diff --git a/vendor/github.com/prometheus/prometheus/tsdb/querier.go b/vendor/github.com/prometheus/prometheus/tsdb/querier.go index 5af688b26db5a..9d7ac8c462de9 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/querier.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/querier.go @@ -85,9 +85,9 @@ func (q *querier) lvals(qs []storage.Querier, n string) ([]string, storage.Warni return mergeStrings(s1, s2), ws, nil } -func (q *querier) Select(sortSeries bool, hints *storage.SelectHints, ms ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (q *querier) Select(sortSeries bool, hints *storage.SelectHints, ms ...*labels.Matcher) storage.SeriesSet { if len(q.blocks) == 0 { - return storage.EmptySeriesSet(), nil, nil + return storage.EmptySeriesSet() } if len(q.blocks) == 1 { // Sorting Head series is slow, and unneeded when only the @@ -96,18 +96,12 @@ func (q *querier) Select(sortSeries bool, hints *storage.SelectHints, ms ...*lab } ss := make([]storage.SeriesSet, len(q.blocks)) - var ws storage.Warnings for i, b := range q.blocks { // We have to sort if blocks > 1 as MergedSeriesSet requires it. - s, w, err := b.Select(true, hints, ms...) - ws = append(ws, w...) - if err != nil { - return nil, ws, err - } - ss[i] = s + ss[i] = b.Select(true, hints, ms...) } - return NewMergedSeriesSet(ss), ws, nil + return NewMergedSeriesSet(ss) } func (q *querier) Close() error { @@ -125,31 +119,23 @@ type verticalQuerier struct { querier } -func (q *verticalQuerier) Select(sortSeries bool, hints *storage.SelectHints, ms ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (q *verticalQuerier) Select(sortSeries bool, hints *storage.SelectHints, ms ...*labels.Matcher) storage.SeriesSet { return q.sel(sortSeries, hints, q.blocks, ms) } -func (q *verticalQuerier) sel(sortSeries bool, hints *storage.SelectHints, qs []storage.Querier, ms []*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (q *verticalQuerier) sel(sortSeries bool, hints *storage.SelectHints, qs []storage.Querier, ms []*labels.Matcher) storage.SeriesSet { if len(qs) == 0 { - return storage.EmptySeriesSet(), nil, nil + return storage.EmptySeriesSet() } if len(qs) == 1 { return qs[0].Select(sortSeries, hints, ms...) } l := len(qs) / 2 - var ws storage.Warnings - a, w, err := q.sel(sortSeries, hints, qs[:l], ms) - ws = append(ws, w...) - if err != nil { - return nil, ws, err - } - b, w, err := q.sel(sortSeries, hints, qs[l:], ms) - ws = append(ws, w...) - if err != nil { - return nil, ws, err - } - return newMergedVerticalSeriesSet(a, b), ws, nil + return newMergedVerticalSeriesSet( + q.sel(sortSeries, hints, qs[:l], ms), + q.sel(sortSeries, hints, qs[l:], ms), + ) } // NewBlockQuerier returns a querier against the reader. @@ -189,7 +175,7 @@ type blockQuerier struct { mint, maxt int64 } -func (q *blockQuerier) Select(sortSeries bool, hints *storage.SelectHints, ms ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (q *blockQuerier) Select(sortSeries bool, hints *storage.SelectHints, ms ...*labels.Matcher) storage.SeriesSet { var base storage.DeprecatedChunkSeriesSet var err error @@ -199,7 +185,7 @@ func (q *blockQuerier) Select(sortSeries bool, hints *storage.SelectHints, ms .. base, err = LookupChunkSeries(q.index, q.tombstones, ms...) } if err != nil { - return nil, nil, err + return storage.ErrSeriesSet(err) } mint := q.mint @@ -218,7 +204,7 @@ func (q *blockQuerier) Select(sortSeries bool, hints *storage.SelectHints, ms .. mint: mint, maxt: maxt, - }, nil, nil + } } func (q *blockQuerier) LabelValues(name string) ([]string, storage.Warnings, error) { @@ -501,6 +487,14 @@ func (s *mergedSeriesSet) Err() error { return s.err } +func (s *mergedSeriesSet) Warnings() storage.Warnings { + var ws storage.Warnings + for _, ss := range s.all { + ws = append(ws, ss.Warnings()...) + } + return ws +} + // nextAll is to call Next() for all SeriesSet. // Because the order of the SeriesSet slice will affect the results, // we need to use an buffer slice to hold the order. @@ -509,7 +503,10 @@ func (s *mergedSeriesSet) nextAll() { for _, ss := range s.all { if ss.Next() { s.buf = append(s.buf, ss) - } else if ss.Err() != nil { + continue + } + + if ss.Err() != nil { s.done = true s.err = ss.Err() break @@ -623,6 +620,13 @@ func (s *mergedVerticalSeriesSet) Err() error { return s.b.Err() } +func (s *mergedVerticalSeriesSet) Warnings() storage.Warnings { + var ws storage.Warnings + ws = append(ws, s.a.Warnings()...) + ws = append(ws, s.b.Warnings()...) + return ws +} + func (s *mergedVerticalSeriesSet) compare() int { if s.adone { return 1 @@ -848,8 +852,9 @@ func (s *blockSeriesSet) Next() bool { return false } -func (s *blockSeriesSet) At() storage.Series { return s.cur } -func (s *blockSeriesSet) Err() error { return s.err } +func (s *blockSeriesSet) At() storage.Series { return s.cur } +func (s *blockSeriesSet) Err() error { return s.err } +func (s *blockSeriesSet) Warnings() storage.Warnings { return nil } // chunkSeries is a series that is backed by a sequence of chunks holding // time series data. diff --git a/vendor/github.com/prometheus/prometheus/web/api/v1/api.go b/vendor/github.com/prometheus/prometheus/web/api/v1/api.go index 8a6e3a9005399..df6d2e658db15 100644 --- a/vendor/github.com/prometheus/prometheus/web/api/v1/api.go +++ b/vendor/github.com/prometheus/prometheus/web/api/v1/api.go @@ -469,7 +469,7 @@ func returnAPIError(err error) *apiError { return nil } - switch err.(type) { + switch errors.Cause(err).(type) { case promql.ErrQueryCanceled: return &apiError{errorCanceled, err} case promql.ErrQueryTimeout: @@ -596,13 +596,8 @@ func (api *API) series(r *http.Request) (result apiFuncResult) { } var sets []storage.SeriesSet - var warnings storage.Warnings for _, mset := range matcherSets { - s, wrn, err := q.Select(false, nil, mset...) - warnings = append(warnings, wrn...) - if err != nil { - return apiFuncResult{nil, &apiError{errorExec, err}, warnings, closer} - } + s := q.Select(false, nil, mset...) sets = append(sets, s) } @@ -611,6 +606,8 @@ func (api *API) series(r *http.Request) (result apiFuncResult) { for set.Next() { metrics = append(metrics, set.At().Labels()) } + + warnings := set.Warnings() if set.Err() != nil { return apiFuncResult{nil, &apiError{errorExec, set.Err()}, warnings, closer} } @@ -1226,12 +1223,9 @@ func (api *API) remoteRead(w http.ResponseWriter, r *http.Request) { return } for i, query := range req.Queries { - err := api.remoteReadQuery(ctx, query, externalLabels, func(querier storage.Querier, hints *storage.SelectHints, filteredMatchers []*labels.Matcher) error { + ws, err := api.remoteReadQuery(ctx, query, externalLabels, func(querier storage.Querier, hints *storage.SelectHints, filteredMatchers []*labels.Matcher) (storage.Warnings, error) { // The streaming API has to provide the series sorted. - set, _, err := querier.Select(true, hints, filteredMatchers...) - if err != nil { - return err - } + set := querier.Select(true, hints, filteredMatchers...) return remote.StreamChunkedReadResponses( remote.NewChunkedWriter(w, f), @@ -1241,6 +1235,9 @@ func (api *API) remoteRead(w http.ResponseWriter, r *http.Request) { api.remoteReadMaxBytesInFrame, ) }) + for _, w := range ws { + level.Warn(api.logger).Log("msg", "warnings on remote read query", "err", w.Error()) + } if err != nil { if httpErr, ok := err.(remote.HTTPError); ok { http.Error(w, httpErr.Error(), httpErr.Status()) @@ -1259,22 +1256,26 @@ func (api *API) remoteRead(w http.ResponseWriter, r *http.Request) { Results: make([]*prompb.QueryResult, len(req.Queries)), } for i, query := range req.Queries { - err := api.remoteReadQuery(ctx, query, externalLabels, func(querier storage.Querier, hints *storage.SelectHints, filteredMatchers []*labels.Matcher) error { - set, _, err := querier.Select(false, hints, filteredMatchers...) - if err != nil { - return err - } + ws, err := api.remoteReadQuery(ctx, query, externalLabels, func(querier storage.Querier, hints *storage.SelectHints, filteredMatchers []*labels.Matcher) (storage.Warnings, error) { + set := querier.Select(false, hints, filteredMatchers...) - resp.Results[i], err = remote.ToQueryResult(set, api.remoteReadSampleLimit) + var ( + ws storage.Warnings + err error + ) + resp.Results[i], ws, err = remote.ToQueryResult(set, api.remoteReadSampleLimit) if err != nil { - return err + return ws, err } for _, ts := range resp.Results[i].Timeseries { ts.Labels = remote.MergeLabels(ts.Labels, sortedExternalLabels) } - return nil + return ws, nil }) + for _, w := range ws { + level.Warn(api.logger).Log("msg", "warnings on remote read query", "err", w.Error()) + } if err != nil { if httpErr, ok := err.(remote.HTTPError); ok { http.Error(w, httpErr.Error(), httpErr.Status()) @@ -1318,15 +1319,15 @@ func filterExtLabelsFromMatchers(pbMatchers []*prompb.LabelMatcher, externalLabe return filteredMatchers, nil } -func (api *API) remoteReadQuery(ctx context.Context, query *prompb.Query, externalLabels map[string]string, seriesHandleFn func(querier storage.Querier, hints *storage.SelectHints, filteredMatchers []*labels.Matcher) error) error { +func (api *API) remoteReadQuery(ctx context.Context, query *prompb.Query, externalLabels map[string]string, seriesHandleFn func(querier storage.Querier, hints *storage.SelectHints, filteredMatchers []*labels.Matcher) (storage.Warnings, error)) (storage.Warnings, error) { filteredMatchers, err := filterExtLabelsFromMatchers(query.Matchers, externalLabels) if err != nil { - return err + return nil, err } querier, err := api.Queryable.Querier(ctx, query.StartTimestampMs, query.EndTimestampMs) if err != nil { - return err + return nil, err } defer func() { if err := querier.Close(); err != nil { diff --git a/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go b/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go index 6d20cfacc55a2..16e528d65e93c 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go +++ b/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go @@ -15,11 +15,12 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + "gopkg.in/yaml.v2" + "github.com/thanos-io/thanos/pkg/discovery/dns" "github.com/thanos-io/thanos/pkg/extprom" "github.com/thanos-io/thanos/pkg/gate" "github.com/thanos-io/thanos/pkg/model" - yaml "gopkg.in/yaml.v2" ) const ( @@ -140,7 +141,7 @@ type memcachedClient struct { asyncQueue chan func() // Gate used to enforce the max number of concurrent GetMulti() operations. - getMultiGate *gate.Gate + getMultiGate gate.Gate // Wait group used to wait all workers on stopping. workers sync.WaitGroup @@ -208,10 +209,8 @@ func newMemcachedClient( dnsProvider: dnsProvider, asyncQueue: make(chan func(), config.MaxAsyncBufferSize), stop: make(chan struct{}, 1), - getMultiGate: gate.NewGate( - config.MaxGetMultiConcurrency, - extprom.WrapRegistererWithPrefix("thanos_memcached_getmulti_", reg), - ), + getMultiGate: gate.NewKeeper(extprom.WrapRegistererWithPrefix("thanos_memcached_getmulti_", reg)). + NewGate(config.MaxGetMultiConcurrency), } c.operations = promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ @@ -270,7 +269,7 @@ func (c *memcachedClient) Stop() { c.workers.Wait() } -func (c *memcachedClient) SetAsync(ctx context.Context, key string, value []byte, ttl time.Duration) error { +func (c *memcachedClient) SetAsync(_ context.Context, key string, value []byte, ttl time.Duration) error { // Skip hitting memcached at all if the item is bigger than the max allowed size. if c.config.MaxItemSize > 0 && uint64(len(value)) > uint64(c.config.MaxItemSize) { c.skipped.WithLabelValues(opSet, reasonMaxItemSize).Inc() @@ -288,7 +287,11 @@ func (c *memcachedClient) SetAsync(ctx context.Context, key string, value []byte }) if err != nil { c.failures.WithLabelValues(opSet).Inc() - level.Warn(c.logger).Log("msg", "failed to store item to memcached", "key", key, "sizeBytes", len(value), "err", err) + + // If the PickServer will fail for any reason the server address will be nil + // and so missing in the logs. We're OK with that (it's a best effort). + serverAddr, _ := c.selector.PickServer(key) + level.Warn(c.logger).Log("msg", "failed to store item to memcached", "key", key, "sizeBytes", len(value), "server", serverAddr, "err", err) return } @@ -388,7 +391,7 @@ func (c *memcachedClient) getMultiSingle(ctx context.Context, keys []string) (it // Wait until we get a free slot from the gate, if the max // concurrency should be enforced. if c.config.MaxGetMultiConcurrency > 0 { - if err := c.getMultiGate.IsMyTurn(ctx); err != nil { + if err := c.getMultiGate.Start(ctx); err != nil { return nil, errors.Wrapf(err, "failed to wait for turn") } defer c.getMultiGate.Done() diff --git a/vendor/github.com/thanos-io/thanos/pkg/compact/compact.go b/vendor/github.com/thanos-io/thanos/pkg/compact/compact.go index 1e74c0738fff3..b034248fbe4d4 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/compact/compact.go +++ b/vendor/github.com/thanos-io/thanos/pkg/compact/compact.go @@ -447,12 +447,15 @@ func (cg *Group) Resolution() int64 { // Compact plans and runs a single compaction against the group. The compacted result // is uploaded into the bucket the blocks were retrieved from. -func (cg *Group) Compact(ctx context.Context, dir string, comp tsdb.Compactor) (bool, ulid.ULID, error) { +func (cg *Group) Compact(ctx context.Context, dir string, comp tsdb.Compactor) (shouldRerun bool, compID ulid.ULID, rerr error) { cg.compactionRunsStarted.Inc() subDir := filepath.Join(dir, cg.Key()) defer func() { + if IsHaltError(rerr) { + return + } if err := os.RemoveAll(subDir); err != nil { level.Error(cg.logger).Log("msg", "failed to remove compaction group work directory", "path", subDir, "err", err) } @@ -879,8 +882,11 @@ func NewBucketCompactor( } // Compact runs compaction over bucket. -func (c *BucketCompactor) Compact(ctx context.Context) error { +func (c *BucketCompactor) Compact(ctx context.Context) (rerr error) { defer func() { + if IsHaltError(rerr) { + return + } if err := os.RemoveAll(c.compactDir); err != nil { level.Error(c.logger).Log("msg", "failed to remove compaction work directory", "path", c.compactDir, "err", err) } diff --git a/vendor/github.com/thanos-io/thanos/pkg/gate/gate.go b/vendor/github.com/thanos-io/thanos/pkg/gate/gate.go index 549b0f3300d4a..4a75df0d5ca66 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/gate/gate.go +++ b/vendor/github.com/thanos-io/thanos/pkg/gate/gate.go @@ -9,56 +9,70 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/prometheus/prometheus/pkg/gate" + promgate "github.com/prometheus/prometheus/pkg/gate" ) -type Gater interface { - IsMyTurn(ctx context.Context) error +// Gate is an interface that mimics prometheus/pkg/gate behaviour. +type Gate interface { + Start(ctx context.Context) error Done() } // Gate wraps the Prometheus gate with extra metrics. -type Gate struct { - g *gate.Gate +type gate struct { + g *promgate.Gate + m *metrics +} + +type metrics struct { inflightQueries prometheus.Gauge gateTiming prometheus.Histogram } -// NewGate returns a new query gate. -func NewGate(maxConcurrent int, reg prometheus.Registerer) *Gate { - g := &Gate{ - g: gate.New(maxConcurrent), - inflightQueries: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ - Name: "gate_queries_in_flight", - Help: "Number of queries that are currently in flight.", - }), - gateTiming: promauto.With(reg).NewHistogram(prometheus.HistogramOpts{ - Name: "gate_duration_seconds", - Help: "How many seconds it took for queries to wait at the gate.", - Buckets: []float64{0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120, 240, 360, 720}, - }), +// Keeper is used to create multiple gates sharing the same metrics. +type Keeper struct { + m *metrics +} + +// NewKeeper creates a new Keeper. +func NewKeeper(reg prometheus.Registerer) *Keeper { + return &Keeper{ + m: &metrics{ + inflightQueries: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ + Name: "gate_queries_in_flight", + Help: "Number of queries that are currently in flight.", + }), + gateTiming: promauto.With(reg).NewHistogram(prometheus.HistogramOpts{ + Name: "gate_duration_seconds", + Help: "How many seconds it took for queries to wait at the gate.", + Buckets: []float64{0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120, 240, 360, 720}, + }), + }, } +} - return g +// NewGate returns a new Gate that collects metrics. +func (k *Keeper) NewGate(maxConcurrent int) Gate { + return &gate{g: promgate.New(maxConcurrent), m: k.m} } -// IsMyTurn iniates a new query and waits until it's our turn to fulfill a query request. -func (g *Gate) IsMyTurn(ctx context.Context) error { +// Start initiates a new request and waits until it's our turn to fulfill a request. +func (g *gate) Start(ctx context.Context) error { start := time.Now() defer func() { - g.gateTiming.Observe(time.Since(start).Seconds()) + g.m.gateTiming.Observe(time.Since(start).Seconds()) }() if err := g.g.Start(ctx); err != nil { return err } - g.inflightQueries.Inc() + g.m.inflightQueries.Inc() return nil } // Done finishes a query. -func (g *Gate) Done() { - g.inflightQueries.Dec() +func (g *gate) Done() { + g.m.inflightQueries.Dec() g.g.Done() } diff --git a/vendor/github.com/thanos-io/thanos/pkg/rules/rulespb/custom.go b/vendor/github.com/thanos-io/thanos/pkg/rules/rulespb/custom.go index 5ffaa21bc4bbe..98205960e468c 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/rules/rulespb/custom.go +++ b/vendor/github.com/thanos-io/thanos/pkg/rules/rulespb/custom.go @@ -5,9 +5,11 @@ package rulespb import ( "encoding/json" + "math/big" "sort" "strconv" "strings" + "time" "github.com/pkg/errors" "github.com/prometheus/prometheus/pkg/labels" @@ -41,12 +43,140 @@ func NewRecordingRule(r *RecordingRule) *Rule { } } +// Compare compares equal recording rules r1 and r2 and returns: +// +// < 0 if r1 < r2 if rule r1 is lexically before rule r2 +// 0 if r1 == r2 +// > 0 if r1 > r2 if rule r1 is lexically after rule r2 +// +// More formally, the ordering is determined in the following order: +// +// 1. recording rule last evaluation (earlier evaluation comes first) +// +// Note: This method assumes r1 and r2 are logically equal as per Rule#Compare. +func (r1 *RecordingRule) Compare(r2 *RecordingRule) int { + if r1.LastEvaluation.Before(r2.LastEvaluation) { + return 1 + } + + if r1.LastEvaluation.After(r2.LastEvaluation) { + return -1 + } + + return 0 +} + func NewAlertingRule(a *Alert) *Rule { return &Rule{ Result: &Rule_Alert{Alert: a}, } } +func (r *Rule) GetLabels() []storepb.Label { + switch { + case r.GetRecording() != nil: + return r.GetRecording().Labels.Labels + case r.GetAlert() != nil: + return r.GetAlert().Labels.Labels + default: + return nil + } +} + +func (r *Rule) SetLabels(ls []storepb.Label) { + var result PromLabels + + if len(ls) > 0 { + result = PromLabels{Labels: ls} + } + + switch { + case r.GetRecording() != nil: + r.GetRecording().Labels = result + case r.GetAlert() != nil: + r.GetAlert().Labels = result + } +} + +func (r *Rule) GetName() string { + switch { + case r.GetRecording() != nil: + return r.GetRecording().Name + case r.GetAlert() != nil: + return r.GetAlert().Name + default: + return "" + } +} + +func (r *Rule) GetQuery() string { + switch { + case r.GetRecording() != nil: + return r.GetRecording().Query + case r.GetAlert() != nil: + return r.GetAlert().Query + default: + return "" + } +} + +func (r *Rule) GetLastEvaluation() time.Time { + switch { + case r.GetRecording() != nil: + return r.GetRecording().LastEvaluation + case r.GetAlert() != nil: + return r.GetAlert().LastEvaluation + default: + return time.Time{} + } +} + +// Compare compares recording and alerting rules r1 and r2 and returns: +// +// < 0 if r1 < r2 if rule r1 is not equal and lexically before rule r2 +// 0 if r1 == r2 if rule r1 is logically equal to r2 (r1 and r2 are the "same" rules) +// > 0 if r1 > r2 if rule r1 is not equal and lexically after rule r2 +// +// More formally, ordering and equality is determined in the following order: +// +// 1. rule type (alerting rules come before recording rules) +// 2. rule name +// 3. rule labels +// 4. rule query +// 5. for alerting rules: duration +// +// Note: this can still leave ordering undetermined for equal rules (x == y). +// For determining ordering of equal rules, use Alert#Compare or RecordingRule#Compare. +func (r1 *Rule) Compare(r2 *Rule) int { + if r1.GetAlert() != nil && r2.GetRecording() != nil { + return -1 + } + + if r1.GetRecording() != nil && r2.GetAlert() != nil { + return 1 + } + + if d := strings.Compare(r1.GetName(), r2.GetName()); d != 0 { + return d + } + + if d := storepb.CompareLabels(r1.GetLabels(), r2.GetLabels()); d != 0 { + return d + } + + if d := strings.Compare(r1.GetQuery(), r2.GetQuery()); d != 0 { + return d + } + + if r1.GetAlert() != nil && r2.GetAlert() != nil { + if d := big.NewFloat(r1.GetAlert().DurationSeconds).Cmp(big.NewFloat(r2.GetAlert().DurationSeconds)); d != 0 { + return d + } + } + + return 0 +} + func (m *Rule) UnmarshalJSON(entry []byte) error { decider := struct { Type string `json:"type"` @@ -120,6 +250,45 @@ func (x *AlertState) MarshalJSON() ([]byte, error) { return []byte(strconv.Quote(x.String())), nil } +// Compare compares alert state x and y and returns: +// +// < 0 if x < y (alert state x is more critical than alert state y) +// 0 if x == y +// > 0 if x > y (alert state x is less critical than alert state y) +// +// For sorting this makes sure that more "critical" alert states come first. +func (x AlertState) Compare(y AlertState) int { + return int(y) - int(x) +} + +// Compare compares two equal alerting rules a1 and a2 and returns: +// +// < 0 if a1 < a2 if rule a1 is lexically before rule a2 +// 0 if a1 == a2 +// > 0 if a1 > a2 if rule a1 is lexically after rule a2 +// +// More formally, the ordering is determined in the following order: +// +// 1. alert state +// 2. alert last evaluation (earlier evaluation comes first) +// +// Note: This method assumes a1 and a2 are logically equal as per Rule#Compare. +func (a1 *Alert) Compare(a2 *Alert) int { + if d := a1.State.Compare(a2.State); d != 0 { + return d + } + + if a1.LastEvaluation.Before(a2.LastEvaluation) { + return 1 + } + + if a1.LastEvaluation.After(a2.LastEvaluation) { + return -1 + } + + return 0 +} + func (m *PromLabels) UnmarshalJSON(entry []byte) error { lbls := labels.Labels{} if err := lbls.UnmarshalJSON(entry); err != nil { diff --git a/vendor/github.com/thanos-io/thanos/pkg/shipper/shipper.go b/vendor/github.com/thanos-io/thanos/pkg/shipper/shipper.go index 46bd9c1bcc03e..e3f1bec55493d 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/shipper/shipper.go +++ b/vendor/github.com/thanos-io/thanos/pkg/shipper/shipper.go @@ -419,7 +419,6 @@ func (s *Shipper) iterBlockMetas(f func(m *metadata.Meta) error) error { return metas[i].BlockMeta.MinTime < metas[j].BlockMeta.MinTime }) for _, m := range metas { - if err := f(m); err != nil { return err } diff --git a/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go b/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go index 85a3a8c246eb2..e5793d78c005b 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go +++ b/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go @@ -31,6 +31,10 @@ import ( "github.com/prometheus/prometheus/tsdb/chunks" "github.com/prometheus/prometheus/tsdb/encoding" "github.com/prometheus/prometheus/tsdb/index" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/indexheader" "github.com/thanos-io/thanos/pkg/block/metadata" @@ -48,9 +52,6 @@ import ( "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/strutil" "github.com/thanos-io/thanos/pkg/tracing" - "golang.org/x/sync/errgroup" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" ) const ( @@ -243,7 +244,7 @@ type BucketStore struct { blockSyncConcurrency int // Query gate which limits the maximum amount of concurrent queries. - queryGate gate.Gater + queryGate gate.Gate // samplesLimiter limits the number of samples per each Series() call. samplesLimiter SampleLimiter @@ -298,21 +299,18 @@ func NewBucketStore( metrics := newBucketStoreMetrics(reg) s := &BucketStore{ - logger: logger, - bkt: bkt, - fetcher: fetcher, - dir: dir, - indexCache: indexCache, - chunkPool: chunkPool, - blocks: map[ulid.ULID]*bucketBlock{}, - blockSets: map[uint64]*bucketBlockSet{}, - debugLogging: debugLogging, - blockSyncConcurrency: blockSyncConcurrency, - filterConfig: filterConfig, - queryGate: gate.NewGate( - maxConcurrent, - extprom.WrapRegistererWithPrefix("thanos_bucket_store_series_", reg), - ), + logger: logger, + bkt: bkt, + fetcher: fetcher, + dir: dir, + indexCache: indexCache, + chunkPool: chunkPool, + blocks: map[ulid.ULID]*bucketBlock{}, + blockSets: map[uint64]*bucketBlockSet{}, + debugLogging: debugLogging, + blockSyncConcurrency: blockSyncConcurrency, + filterConfig: filterConfig, + queryGate: gate.NewKeeper(extprom.WrapRegistererWithPrefix("thanos_bucket_store_series_", reg)).NewGate(maxConcurrent), samplesLimiter: NewLimiter(maxSampleCount, metrics.queriesDropped), partitioner: gapBasedPartitioner{maxGapSize: partitionerMaxGapSize}, enableCompatibilityLabel: enableCompatibilityLabel, @@ -847,7 +845,7 @@ func debugFoundBlockSetOverview(logger log.Logger, mint, maxt, maxResolutionMill // Series implements the storepb.StoreServer interface. func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storepb.Store_SeriesServer) (err error) { tracing.DoInSpan(srv.Context(), "store_query_gate_ismyturn", func(ctx context.Context) { - err = s.queryGate.IsMyTurn(srv.Context()) + err = s.queryGate.Start(srv.Context()) }) if err != nil { return errors.Wrapf(err, "failed to wait for turn") @@ -984,9 +982,8 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storepb.Store_Serie tracing.DoInSpan(ctx, "bucket_store_merge_all", func(ctx context.Context) { begin := time.Now() - // Merge series set into an union of all block sets. This exposes all blocks are single seriesSet. - // Chunks of returned series might be out of order w.r.t to their time range. - // This must be accounted for later by clients. + // NOTE: We "carefully" assume series and chunks are sorted within each SeriesSet. This should be guaranteed by + // blockSeries method. In worst case deduplication logic won't deduplicate correctly, which will be accounted later. set := storepb.MergeSeriesSets(res...) for set.Next() { var series storepb.Series diff --git a/vendor/github.com/thanos-io/thanos/pkg/store/multitsdb.go b/vendor/github.com/thanos-io/thanos/pkg/store/multitsdb.go index e1acea5514cac..b55535b036646 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/store/multitsdb.go +++ b/vendor/github.com/thanos-io/thanos/pkg/store/multitsdb.go @@ -13,12 +13,14 @@ import ( "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "github.com/thanos-io/thanos/pkg/component" - "github.com/thanos-io/thanos/pkg/store/storepb" "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + + "github.com/thanos-io/thanos/pkg/component" + "github.com/thanos-io/thanos/pkg/store/storepb" + "github.com/thanos-io/thanos/pkg/tracing" ) // MultiTSDBStore implements the Store interface backed by multiple TSDBStore instances. @@ -113,7 +115,11 @@ func (s *tenantSeriesSetServer) Context() context.Context { } func (s *tenantSeriesSetServer) Series(store *TSDBStore, r *storepb.SeriesRequest) { - err := store.Series(r, s) + var err error + tracing.DoInSpan(s.ctx, "multitsdb_tenant_series", func(_ context.Context) { + err = store.Series(r, s) + }) + if err != nil { if r.PartialResponseDisabled { s.err = errors.Wrapf(err, "get series for tenant %s", s.tenant) @@ -164,12 +170,13 @@ func (s *MultiTSDBStore) Series(r *storepb.SeriesRequest, srv storepb.Store_Seri } var ( - g, gctx = errgroup.WithContext(srv.Context()) - + g, gctx = errgroup.WithContext(srv.Context()) + span, ctx = tracing.StartSpan(gctx, "multitsdb_series") // Allow to buffer max 10 series response. // Each might be quite large (multi chunk long series given by sidecar). respSender, respRecv, closeFn = newRespCh(gctx, 10) ) + defer span.Finish() g.Go(func() error { var ( @@ -184,7 +191,7 @@ func (s *MultiTSDBStore) Series(r *storepb.SeriesRequest, srv storepb.Store_Seri for tenant, store := range stores { store := store - seriesCtx, cancelSeries := context.WithCancel(gctx) + seriesCtx, cancelSeries := context.WithCancel(ctx) seriesCtx = grpc_opentracing.ClientAddContextTags(seriesCtx, opentracing.Tags{ "tenant": tenant, }) @@ -219,6 +226,9 @@ func (s *MultiTSDBStore) Series(r *storepb.SeriesRequest, srv storepb.Store_Seri // LabelNames returns all known label names. func (s *MultiTSDBStore) LabelNames(ctx context.Context, req *storepb.LabelNamesRequest) (*storepb.LabelNamesResponse, error) { + span, ctx := tracing.StartSpan(ctx, "multitsdb_label_names") + defer span.Finish() + names := map[string]struct{}{} warnings := map[string]struct{}{} @@ -259,6 +269,9 @@ func keys(m map[string]struct{}) []string { // LabelValues returns all known label values for a given label name. func (s *MultiTSDBStore) LabelValues(ctx context.Context, req *storepb.LabelValuesRequest) (*storepb.LabelValuesResponse, error) { + span, ctx := tracing.StartSpan(ctx, "multitsdb_label_values") + defer span.Finish() + values := map[string]struct{}{} warnings := map[string]struct{}{} diff --git a/vendor/github.com/thanos-io/thanos/pkg/store/proxy.go b/vendor/github.com/thanos-io/thanos/pkg/store/proxy.go index 840c391bd00df..b5c310920edb4 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/store/proxy.go +++ b/vendor/github.com/thanos-io/thanos/pkg/store/proxy.go @@ -474,6 +474,7 @@ func (s *streamSeriesSet) At() ([]storepb.Label, []storepb.AggrChunk) { } return s.currSeries.Labels, s.currSeries.Chunks } + func (s *streamSeriesSet) Err() error { s.errMtx.Lock() defer s.errMtx.Unlock() diff --git a/vendor/github.com/thanos-io/thanos/pkg/store/storepb/custom.go b/vendor/github.com/thanos-io/thanos/pkg/store/storepb/custom.go index 66448324b8859..61214a611fba6 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/store/storepb/custom.go +++ b/vendor/github.com/thanos-io/thanos/pkg/store/storepb/custom.go @@ -4,6 +4,7 @@ package storepb import ( + "bytes" "fmt" "sort" "strconv" @@ -50,6 +51,7 @@ func NewHintsSeriesResponse(hints *types.Any) *SeriesResponse { } // CompareLabels compares two sets of labels. +// After lexicographical order, the set with fewer labels comes first. func CompareLabels(a, b []Label) int { l := len(a) if len(b) < l { @@ -63,7 +65,7 @@ func CompareLabels(a, b []Label) int { return d } } - // If all labels so far were in common, the set with fewer labels comes first. + return len(a) - len(b) } @@ -78,13 +80,25 @@ func EmptySeriesSet() SeriesSet { return emptySeriesSet{} } -// MergeSeriesSets returns a new series set that is the union of the input sets. +// MergeSeriesSets takes all series sets and returns as a union single series set. +// It assumes series are sorted by labels within single SeriesSet, similar to remote read guarantees. +// However, they can be partial: in such case, if the single SeriesSet returns the same series within many iterations, +// MergeSeriesSets will merge those into one. +// +// It also assumes in a "best effort" way that chunks are sorted by min time. It's done as an optimization only, so if input +// series' chunks are NOT sorted, the only consequence is that the duplicates might be not correctly removed. This is double checked +// which on just-before PromQL level as well, so the only consequence is increased network bandwidth. +// If all chunks were sorted, MergeSeriesSet ALSO returns sorted chunks by min time. +// +// Chunks within the same series can also overlap (within all SeriesSet +// as well as single SeriesSet alone). If the chunk ranges overlap, the *exact* chunk duplicates will be removed +// (except one), and any other overlaps will be appended into on chunks slice. func MergeSeriesSets(all ...SeriesSet) SeriesSet { switch len(all) { case 0: return emptySeriesSet{} case 1: - return all[0] + return newUniqueSeriesSet(all[0]) } h := len(all) / 2 @@ -111,11 +125,6 @@ type mergedSeriesSet struct { adone, bdone bool } -// newMergedSeriesSet takes two series sets as a single series set. -// Series that occur in both sets should have disjoint time ranges. -// If the ranges overlap b samples are appended to a samples. -// If the single SeriesSet returns same series within many iterations, -// merge series set will not try to merge those. func newMergedSeriesSet(a, b SeriesSet) *mergedSeriesSet { s := &mergedSeriesSet{a: a, b: b} // Initialize first elements of both sets as Next() needs @@ -155,33 +164,175 @@ func (s *mergedSeriesSet) Next() bool { } d := s.compare() - - // Both sets contain the current series. Chain them into a single one. if d > 0 { s.lset, s.chunks = s.b.At() s.bdone = !s.b.Next() - } else if d < 0 { + return true + } + if d < 0 { s.lset, s.chunks = s.a.At() s.adone = !s.a.Next() - } else { - // Concatenate chunks from both series sets. They may be expected of order - // w.r.t to their time range. This must be accounted for later. - lset, chksA := s.a.At() - _, chksB := s.b.At() - - s.lset = lset - // Slice reuse is not generally safe with nested merge iterators. - // We err on the safe side an create a new slice. - s.chunks = make([]AggrChunk, 0, len(chksA)+len(chksB)) - s.chunks = append(s.chunks, chksA...) - s.chunks = append(s.chunks, chksB...) + return true + } - s.adone = !s.a.Next() - s.bdone = !s.b.Next() + // Both a and b contains the same series. Go through all chunks, remove duplicates and concatenate chunks from both + // series sets. We best effortly assume chunks are sorted by min time. If not, we will not detect all deduplicate which will + // be account on select layer anyway. We do it still for early optimization. + lset, chksA := s.a.At() + _, chksB := s.b.At() + s.lset = lset + + // Slice reuse is not generally safe with nested merge iterators. + // We err on the safe side an create a new slice. + s.chunks = make([]AggrChunk, 0, len(chksA)+len(chksB)) + + b := 0 +Outer: + for a := range chksA { + for { + if b >= len(chksB) { + // No more b chunks. + s.chunks = append(s.chunks, chksA[a:]...) + break Outer + } + + cmp := chksA[a].Compare(chksB[b]) + if cmp > 0 { + s.chunks = append(s.chunks, chksA[a]) + break + } + if cmp < 0 { + s.chunks = append(s.chunks, chksB[b]) + b++ + continue + } + + // Exact duplicated chunks, discard one from b. + b++ + } } + + if b < len(chksB) { + s.chunks = append(s.chunks, chksB[b:]...) + } + + s.adone = !s.a.Next() + s.bdone = !s.b.Next() return true } +// uniqueSeriesSet takes one series set and ensures each iteration contains single, full series. +type uniqueSeriesSet struct { + SeriesSet + done bool + + peek *Series + + lset []Label + chunks []AggrChunk +} + +func newUniqueSeriesSet(wrapped SeriesSet) *uniqueSeriesSet { + return &uniqueSeriesSet{SeriesSet: wrapped} +} + +func (s *uniqueSeriesSet) At() ([]Label, []AggrChunk) { + return s.lset, s.chunks +} + +func (s *uniqueSeriesSet) Next() bool { + if s.Err() != nil { + return false + } + + for !s.done { + if s.done = !s.SeriesSet.Next(); s.done { + break + } + lset, chks := s.SeriesSet.At() + if s.peek == nil { + s.peek = &Series{Labels: lset, Chunks: chks} + continue + } + + if CompareLabels(lset, s.peek.Labels) != 0 { + s.lset, s.chunks = s.peek.Labels, s.peek.Chunks + s.peek = &Series{Labels: lset, Chunks: chks} + return true + } + + // We assume non-overlapping, sorted chunks. This is best effort only, if it's otherwise it + // will just be duplicated, but well handled by StoreAPI consumers. + s.peek.Chunks = append(s.peek.Chunks, chks...) + } + + if s.peek == nil { + return false + } + + s.lset, s.chunks = s.peek.Labels, s.peek.Chunks + s.peek = nil + return true +} + +// Compare returns positive 1 if chunk is smaller -1 if larger than b by min time, then max time. +// It returns 0 if chunks are exactly the same. +func (m AggrChunk) Compare(b AggrChunk) int { + if m.MinTime < b.MinTime { + return 1 + } + if m.MinTime > b.MinTime { + return -1 + } + + // Same min time. + if m.MaxTime < b.MaxTime { + return 1 + } + if m.MaxTime > b.MaxTime { + return -1 + } + + // We could use proto.Equal, but we need ordering as well. + for _, cmp := range []func() int{ + func() int { return m.Raw.Compare(b.Raw) }, + func() int { return m.Count.Compare(b.Count) }, + func() int { return m.Sum.Compare(b.Sum) }, + func() int { return m.Min.Compare(b.Min) }, + func() int { return m.Max.Compare(b.Max) }, + func() int { return m.Counter.Compare(b.Counter) }, + } { + if c := cmp(); c == 0 { + continue + } else { + return c + } + } + return 0 +} + +// Compare returns positive 1 if chunk is smaller -1 if larger. +// It returns 0 if chunks are exactly the same. +func (m *Chunk) Compare(b *Chunk) int { + if m == nil && b == nil { + return 0 + } + if b == nil { + return 1 + } + if m == nil { + return -1 + } + + if m.Type < b.Type { + return 1 + } + if m.Type > b.Type { + return -1 + } + return bytes.Compare(m.Data, b.Data) +} + // LabelsToPromLabels converts Thanos proto labels to Prometheus labels in type safe manner. // NOTE: It allocates memory. func LabelsToPromLabels(lset []Label) labels.Labels { diff --git a/vendor/github.com/thanos-io/thanos/pkg/store/storepb/rpc.pb.go b/vendor/github.com/thanos-io/thanos/pkg/store/storepb/rpc.pb.go index 51412902baf7d..bddab10d767b8 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/store/storepb/rpc.pb.go +++ b/vendor/github.com/thanos-io/thanos/pkg/store/storepb/rpc.pb.go @@ -707,6 +707,9 @@ type StoreClient interface { /// partition of the single series, but once a new series is started to be streamed it means that no more data will /// be sent for previous one. /// Series has to be sorted. + /// + /// There is no requirements on chunk sorting, however it is recommended to have chunk sorted by chunk min time. + /// This heavily optimizes the resource usage on Querier / Federated Queries. Series(ctx context.Context, in *SeriesRequest, opts ...grpc.CallOption) (Store_SeriesClient, error) /// LabelNames returns all label names that is available. /// Currently unimplemented in all Thanos implementations, because Query API does not implement this either. @@ -793,6 +796,9 @@ type StoreServer interface { /// partition of the single series, but once a new series is started to be streamed it means that no more data will /// be sent for previous one. /// Series has to be sorted. + /// + /// There is no requirements on chunk sorting, however it is recommended to have chunk sorted by chunk min time. + /// This heavily optimizes the resource usage on Querier / Federated Queries. Series(*SeriesRequest, Store_SeriesServer) error /// LabelNames returns all label names that is available. /// Currently unimplemented in all Thanos implementations, because Query API does not implement this either. diff --git a/vendor/github.com/thanos-io/thanos/pkg/store/storepb/rpc.proto b/vendor/github.com/thanos-io/thanos/pkg/store/storepb/rpc.proto index 94ef67b7d7ea6..51a53c88d97f6 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/store/storepb/rpc.proto +++ b/vendor/github.com/thanos-io/thanos/pkg/store/storepb/rpc.proto @@ -34,6 +34,9 @@ service Store { /// partition of the single series, but once a new series is started to be streamed it means that no more data will /// be sent for previous one. /// Series has to be sorted. + /// + /// There is no requirements on chunk sorting, however it is recommended to have chunk sorted by chunk min time. + /// This heavily optimizes the resource usage on Querier / Federated Queries. rpc Series(SeriesRequest) returns (stream SeriesResponse); /// LabelNames returns all label names that is available. diff --git a/vendor/github.com/thanos-io/thanos/pkg/store/tsdb.go b/vendor/github.com/thanos-io/thanos/pkg/store/tsdb.go index a1ead860a4fe1..b06cb3ac3efd1 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/store/tsdb.go +++ b/vendor/github.com/thanos-io/thanos/pkg/store/tsdb.go @@ -14,12 +14,13 @@ import ( "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/chunkenc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/promclient" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/store/storepb" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" ) // TSDBStore implements the store API against a local TSDB instance. @@ -107,13 +108,10 @@ func (s *TSDBStore) Series(r *storepb.SeriesRequest, srv storepb.Store_SeriesSer } defer runutil.CloseWithLogOnErr(s.logger, q, "close tsdb querier series") - set, _, err := q.Select(false, nil, matchers...) - if err != nil { - return status.Error(codes.Internal, err.Error()) - } - - var respSeries storepb.Series - + var ( + set = q.Select(false, nil, matchers...) + respSeries storepb.Series + ) for set.Next() { series := set.At() @@ -139,6 +137,9 @@ func (s *TSDBStore) Series(r *storepb.SeriesRequest, srv storepb.Store_SeriesSer return status.Error(codes.Aborted, err.Error()) } } + if err := set.Err(); err != nil { + return status.Error(codes.Internal, err.Error()) + } return nil } @@ -214,7 +215,7 @@ func (s *TSDBStore) translateAndExtendLabels(m, extend labels.Labels) []storepb. func (s *TSDBStore) LabelNames(ctx context.Context, _ *storepb.LabelNamesRequest) ( *storepb.LabelNamesResponse, error, ) { - q, err := s.db.Querier(context.Background(), math.MinInt64, math.MaxInt64) + q, err := s.db.Querier(ctx, math.MinInt64, math.MaxInt64) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } @@ -231,7 +232,7 @@ func (s *TSDBStore) LabelNames(ctx context.Context, _ *storepb.LabelNamesRequest func (s *TSDBStore) LabelValues(ctx context.Context, r *storepb.LabelValuesRequest) ( *storepb.LabelValuesResponse, error, ) { - q, err := s.db.Querier(context.Background(), math.MinInt64, math.MaxInt64) + q, err := s.db.Querier(ctx, math.MinInt64, math.MaxInt64) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go b/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go index 4c07314534997..ec9709737ada5 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go +++ b/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go @@ -29,6 +29,7 @@ func ContextWithTracer(ctx context.Context, tracer opentracing.Tracer) context.C return context.WithValue(ctx, tracerKey, tracer) } +// tracerFromContext extracts opentracing.Tracer from the given context. func tracerFromContext(ctx context.Context) opentracing.Tracer { val := ctx.Value(tracerKey) if sp, ok := val.(opentracing.Tracer); ok { @@ -37,6 +38,15 @@ func tracerFromContext(ctx context.Context) opentracing.Tracer { return nil } +// CopyTraceContext copies the necessary trace context from given source context to target context. +func CopyTraceContext(trgt, src context.Context) context.Context { + ctx := ContextWithTracer(trgt, tracerFromContext(src)) + if parentSpan := opentracing.SpanFromContext(src); parentSpan != nil { + ctx = opentracing.ContextWithSpan(ctx, parentSpan) + } + return ctx +} + // StartSpan starts and returns span with `operationName` and hooking as child to a span found within given context if any. // It uses opentracing.Tracer propagated in context. If no found, it uses noop tracer without notification. func StartSpan(ctx context.Context, operationName string, opts ...opentracing.StartSpanOption) (opentracing.Span, context.Context) { diff --git a/vendor/go.etcd.io/bbolt/README.md b/vendor/go.etcd.io/bbolt/README.md index 2dff3761da36b..c9e64b1a6150b 100644 --- a/vendor/go.etcd.io/bbolt/README.md +++ b/vendor/go.etcd.io/bbolt/README.md @@ -152,11 +152,12 @@ are not thread safe. To work with data in multiple goroutines you must start a transaction for each one or use locking to ensure only one goroutine accesses a transaction at a time. Creating transaction from the `DB` is thread safe. -Read-only transactions and read-write transactions should not depend on one -another and generally shouldn't be opened simultaneously in the same goroutine. -This can cause a deadlock as the read-write transaction needs to periodically -re-map the data file but it cannot do so while a read-only transaction is open. - +Transactions should not depend on one another and generally shouldn't be opened +simultaneously in the same goroutine. This can cause a deadlock as the read-write +transaction needs to periodically re-map the data file but it cannot do so while +any read-only transaction is open. Even a nested read-only transaction can cause +a deadlock, as the child transaction can block the parent transaction from releasing +its resources. #### Read-write transactions diff --git a/vendor/go.etcd.io/bbolt/freelist.go b/vendor/go.etcd.io/bbolt/freelist.go index d441b69256dad..697a46968bacf 100644 --- a/vendor/go.etcd.io/bbolt/freelist.go +++ b/vendor/go.etcd.io/bbolt/freelist.go @@ -2,7 +2,6 @@ package bbolt import ( "fmt" - "reflect" "sort" "unsafe" ) @@ -94,24 +93,8 @@ func (f *freelist) pending_count() int { return count } -// copyallunsafe copies a list of all free ids and all pending ids in one sorted list. +// copyall copies a list of all free ids and all pending ids in one sorted list. // f.count returns the minimum length required for dst. -func (f *freelist) copyallunsafe(dstptr unsafe.Pointer) { // dstptr is []pgid data pointer - m := make(pgids, 0, f.pending_count()) - for _, txp := range f.pending { - m = append(m, txp.ids...) - } - sort.Sort(m) - fpgids := f.getFreePageIDs() - sz := len(fpgids) + len(m) - dst := *(*[]pgid)(unsafe.Pointer(&reflect.SliceHeader{ - Data: uintptr(dstptr), - Len: sz, - Cap: sz, - })) - mergepgids(dst, fpgids, m) -} - func (f *freelist) copyall(dst []pgid) { m := make(pgids, 0, f.pending_count()) for _, txp := range f.pending { @@ -284,21 +267,23 @@ func (f *freelist) read(p *page) { } // If the page.count is at the max uint16 value (64k) then it's considered // an overflow and the size of the freelist is stored as the first element. - var idx, count uintptr = 0, uintptr(p.count) + var idx, count = 0, int(p.count) if count == 0xFFFF { idx = 1 - count = uintptr(*(*pgid)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + unsafe.Sizeof(*p)))) + c := *(*pgid)(unsafeAdd(unsafe.Pointer(p), unsafe.Sizeof(*p))) + count = int(c) + if count < 0 { + panic(fmt.Sprintf("leading element count %d overflows int", c)) + } } // Copy the list of page ids from the freelist. if count == 0 { f.ids = nil } else { - ids := *(*[]pgid)(unsafe.Pointer(&reflect.SliceHeader{ - Data: uintptr(unsafe.Pointer(p)) + unsafe.Sizeof(*p) + idx*unsafe.Sizeof(pgid(0)), - Len: int(count), - Cap: int(count), - })) + var ids []pgid + data := unsafeIndex(unsafe.Pointer(p), unsafe.Sizeof(*p), unsafe.Sizeof(ids[0]), idx) + unsafeSlice(unsafe.Pointer(&ids), data, count) // copy the ids, so we don't modify on the freelist page directly idsCopy := make([]pgid, count) @@ -331,16 +316,22 @@ func (f *freelist) write(p *page) error { // The page.count can only hold up to 64k elements so if we overflow that // number then we handle it by putting the size in the first element. - lenids := f.count() - if lenids == 0 { - p.count = uint16(lenids) - } else if lenids < 0xFFFF { - p.count = uint16(lenids) - f.copyallunsafe(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + unsafe.Sizeof(*p))) + l := f.count() + if l == 0 { + p.count = uint16(l) + } else if l < 0xFFFF { + p.count = uint16(l) + var ids []pgid + data := unsafeAdd(unsafe.Pointer(p), unsafe.Sizeof(*p)) + unsafeSlice(unsafe.Pointer(&ids), data, l) + f.copyall(ids) } else { p.count = 0xFFFF - *(*pgid)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + unsafe.Sizeof(*p))) = pgid(lenids) - f.copyallunsafe(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + unsafe.Sizeof(*p) + unsafe.Sizeof(pgid(0)))) + var ids []pgid + data := unsafeAdd(unsafe.Pointer(p), unsafe.Sizeof(*p)) + unsafeSlice(unsafe.Pointer(&ids), data, l+1) + ids[0] = pgid(l) + f.copyall(ids[1:]) } return nil diff --git a/vendor/go.etcd.io/bbolt/node.go b/vendor/go.etcd.io/bbolt/node.go index 1690eef3f7a49..73988b5c4c0ac 100644 --- a/vendor/go.etcd.io/bbolt/node.go +++ b/vendor/go.etcd.io/bbolt/node.go @@ -3,7 +3,6 @@ package bbolt import ( "bytes" "fmt" - "reflect" "sort" "unsafe" ) @@ -208,36 +207,32 @@ func (n *node) write(p *page) { } // Loop over each item and write it to the page. - bp := uintptr(unsafe.Pointer(p)) + unsafe.Sizeof(*p) + n.pageElementSize()*uintptr(len(n.inodes)) + // off tracks the offset into p of the start of the next data. + off := unsafe.Sizeof(*p) + n.pageElementSize()*uintptr(len(n.inodes)) for i, item := range n.inodes { _assert(len(item.key) > 0, "write: zero-length inode key") + // Create a slice to write into of needed size and advance + // byte pointer for next iteration. + sz := len(item.key) + len(item.value) + b := unsafeByteSlice(unsafe.Pointer(p), off, 0, sz) + off += uintptr(sz) + // Write the page element. if n.isLeaf { elem := p.leafPageElement(uint16(i)) - elem.pos = uint32(bp - uintptr(unsafe.Pointer(elem))) + elem.pos = uint32(uintptr(unsafe.Pointer(&b[0])) - uintptr(unsafe.Pointer(elem))) elem.flags = item.flags elem.ksize = uint32(len(item.key)) elem.vsize = uint32(len(item.value)) } else { elem := p.branchPageElement(uint16(i)) - elem.pos = uint32(bp - uintptr(unsafe.Pointer(elem))) + elem.pos = uint32(uintptr(unsafe.Pointer(&b[0])) - uintptr(unsafe.Pointer(elem))) elem.ksize = uint32(len(item.key)) elem.pgid = item.pgid _assert(elem.pgid != p.id, "write: circular dependency occurred") } - // Create a slice to write into of needed size and advance - // byte pointer for next iteration. - klen, vlen := len(item.key), len(item.value) - sz := klen + vlen - b := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ - Data: bp, - Len: sz, - Cap: sz, - })) - bp += uintptr(sz) - // Write data for the element to the end of the page. l := copy(b, item.key) copy(b[l:], item.value) diff --git a/vendor/go.etcd.io/bbolt/page.go b/vendor/go.etcd.io/bbolt/page.go index b5c1699789180..c9a158fb066c0 100644 --- a/vendor/go.etcd.io/bbolt/page.go +++ b/vendor/go.etcd.io/bbolt/page.go @@ -3,7 +3,6 @@ package bbolt import ( "fmt" "os" - "reflect" "sort" "unsafe" ) @@ -51,13 +50,13 @@ func (p *page) typ() string { // meta returns a pointer to the metadata section of the page. func (p *page) meta() *meta { - return (*meta)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + unsafe.Sizeof(*p))) + return (*meta)(unsafeAdd(unsafe.Pointer(p), unsafe.Sizeof(*p))) } // leafPageElement retrieves the leaf node by index func (p *page) leafPageElement(index uint16) *leafPageElement { - off := uintptr(index) * unsafe.Sizeof(leafPageElement{}) - return (*leafPageElement)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + unsafe.Sizeof(*p) + off)) + return (*leafPageElement)(unsafeIndex(unsafe.Pointer(p), unsafe.Sizeof(*p), + leafPageElementSize, int(index))) } // leafPageElements retrieves a list of leaf nodes. @@ -65,17 +64,16 @@ func (p *page) leafPageElements() []leafPageElement { if p.count == 0 { return nil } - return *(*[]leafPageElement)(unsafe.Pointer(&reflect.SliceHeader{ - Data: uintptr(unsafe.Pointer(p)) + unsafe.Sizeof(*p), - Len: int(p.count), - Cap: int(p.count), - })) + var elems []leafPageElement + data := unsafeAdd(unsafe.Pointer(p), unsafe.Sizeof(*p)) + unsafeSlice(unsafe.Pointer(&elems), data, int(p.count)) + return elems } // branchPageElement retrieves the branch node by index func (p *page) branchPageElement(index uint16) *branchPageElement { - off := uintptr(index) * unsafe.Sizeof(branchPageElement{}) - return (*branchPageElement)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + unsafe.Sizeof(*p) + off)) + return (*branchPageElement)(unsafeIndex(unsafe.Pointer(p), unsafe.Sizeof(*p), + unsafe.Sizeof(branchPageElement{}), int(index))) } // branchPageElements retrieves a list of branch nodes. @@ -83,20 +81,15 @@ func (p *page) branchPageElements() []branchPageElement { if p.count == 0 { return nil } - return *(*[]branchPageElement)(unsafe.Pointer(&reflect.SliceHeader{ - Data: uintptr(unsafe.Pointer(p)) + unsafe.Sizeof(*p), - Len: int(p.count), - Cap: int(p.count), - })) + var elems []branchPageElement + data := unsafeAdd(unsafe.Pointer(p), unsafe.Sizeof(*p)) + unsafeSlice(unsafe.Pointer(&elems), data, int(p.count)) + return elems } // dump writes n bytes of the page to STDERR as hex output. func (p *page) hexdump(n int) { - buf := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ - Data: uintptr(unsafe.Pointer(p)), - Len: n, - Cap: n, - })) + buf := unsafeByteSlice(unsafe.Pointer(p), 0, 0, n) fmt.Fprintf(os.Stderr, "%x\n", buf) } @@ -115,11 +108,7 @@ type branchPageElement struct { // key returns a byte slice of the node key. func (n *branchPageElement) key() []byte { - return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ - Data: uintptr(unsafe.Pointer(n)) + uintptr(n.pos), - Len: int(n.ksize), - Cap: int(n.ksize), - })) + return unsafeByteSlice(unsafe.Pointer(n), 0, int(n.pos), int(n.pos)+int(n.ksize)) } // leafPageElement represents a node on a leaf page. @@ -132,20 +121,16 @@ type leafPageElement struct { // key returns a byte slice of the node key. func (n *leafPageElement) key() []byte { - return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ - Data: uintptr(unsafe.Pointer(n)) + uintptr(n.pos), - Len: int(n.ksize), - Cap: int(n.ksize), - })) + i := int(n.pos) + j := i + int(n.ksize) + return unsafeByteSlice(unsafe.Pointer(n), 0, i, j) } // value returns a byte slice of the node value. func (n *leafPageElement) value() []byte { - return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ - Data: uintptr(unsafe.Pointer(n)) + uintptr(n.pos) + uintptr(n.ksize), - Len: int(n.vsize), - Cap: int(n.vsize), - })) + i := int(n.pos) + int(n.ksize) + j := i + int(n.vsize) + return unsafeByteSlice(unsafe.Pointer(n), 0, i, j) } // PageInfo represents human readable information about a page. diff --git a/vendor/go.etcd.io/bbolt/tx.go b/vendor/go.etcd.io/bbolt/tx.go index 13937cdbf612a..4b1a64a8b8aa6 100644 --- a/vendor/go.etcd.io/bbolt/tx.go +++ b/vendor/go.etcd.io/bbolt/tx.go @@ -4,7 +4,6 @@ import ( "fmt" "io" "os" - "reflect" "sort" "strings" "time" @@ -524,24 +523,18 @@ func (tx *Tx) write() error { // Write pages to disk in order. for _, p := range pages { - size := (int(p.overflow) + 1) * tx.db.pageSize + rem := (uint64(p.overflow) + 1) * uint64(tx.db.pageSize) offset := int64(p.id) * int64(tx.db.pageSize) + var written uintptr // Write out page in "max allocation" sized chunks. - ptr := uintptr(unsafe.Pointer(p)) for { - // Limit our write to our max allocation size. - sz := size + sz := rem if sz > maxAllocSize-1 { sz = maxAllocSize - 1 } + buf := unsafeByteSlice(unsafe.Pointer(p), written, 0, int(sz)) - // Write chunk to disk. - buf := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ - Data: ptr, - Len: sz, - Cap: sz, - })) if _, err := tx.db.ops.writeAt(buf, offset); err != nil { return err } @@ -550,14 +543,14 @@ func (tx *Tx) write() error { tx.stats.Write++ // Exit inner for loop if we've written all the chunks. - size -= sz - if size == 0 { + rem -= sz + if rem == 0 { break } // Otherwise move offset forward and move pointer to next chunk. offset += int64(sz) - ptr += uintptr(sz) + written += uintptr(sz) } } @@ -576,11 +569,7 @@ func (tx *Tx) write() error { continue } - buf := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ - Data: uintptr(unsafe.Pointer(p)), - Len: tx.db.pageSize, - Cap: tx.db.pageSize, - })) + buf := unsafeByteSlice(unsafe.Pointer(p), 0, 0, tx.db.pageSize) // See https://go.googlesource.com/go/+/f03c9202c43e0abb130669852082117ca50aa9b1 for i := range buf { diff --git a/vendor/go.etcd.io/bbolt/unsafe.go b/vendor/go.etcd.io/bbolt/unsafe.go new file mode 100644 index 0000000000000..c0e50375007fa --- /dev/null +++ b/vendor/go.etcd.io/bbolt/unsafe.go @@ -0,0 +1,39 @@ +package bbolt + +import ( + "reflect" + "unsafe" +) + +func unsafeAdd(base unsafe.Pointer, offset uintptr) unsafe.Pointer { + return unsafe.Pointer(uintptr(base) + offset) +} + +func unsafeIndex(base unsafe.Pointer, offset uintptr, elemsz uintptr, n int) unsafe.Pointer { + return unsafe.Pointer(uintptr(base) + offset + uintptr(n)*elemsz) +} + +func unsafeByteSlice(base unsafe.Pointer, offset uintptr, i, j int) []byte { + // See: https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices + // + // This memory is not allocated from C, but it is unmanaged by Go's + // garbage collector and should behave similarly, and the compiler + // should produce similar code. Note that this conversion allows a + // subslice to begin after the base address, with an optional offset, + // while the URL above does not cover this case and only slices from + // index 0. However, the wiki never says that the address must be to + // the beginning of a C allocation (or even that malloc was used at + // all), so this is believed to be correct. + return (*[maxAllocSize]byte)(unsafeAdd(base, offset))[i:j:j] +} + +// unsafeSlice modifies the data, len, and cap of a slice variable pointed to by +// the slice parameter. This helper should be used over other direct +// manipulation of reflect.SliceHeader to prevent misuse, namely, converting +// from reflect.SliceHeader to a Go slice type. +func unsafeSlice(slice, data unsafe.Pointer, len int) { + s := (*reflect.SliceHeader)(slice) + s.Data = uintptr(data) + s.Cap = len + s.Len = len +} diff --git a/vendor/golang.org/x/sys/cpu/byteorder.go b/vendor/golang.org/x/sys/cpu/byteorder.go index ed8da8deacf08..dcbb14ef35a48 100644 --- a/vendor/golang.org/x/sys/cpu/byteorder.go +++ b/vendor/golang.org/x/sys/cpu/byteorder.go @@ -39,20 +39,25 @@ func (bigEndian) Uint64(b []byte) uint64 { uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 } -// hostByteOrder returns binary.LittleEndian on little-endian machines and -// binary.BigEndian on big-endian machines. +// hostByteOrder returns littleEndian on little-endian machines and +// bigEndian on big-endian machines. func hostByteOrder() byteOrder { switch runtime.GOARCH { case "386", "amd64", "amd64p32", + "alpha", "arm", "arm64", "mipsle", "mips64le", "mips64p32le", + "nios2", "ppc64le", - "riscv", "riscv64": + "riscv", "riscv64", + "sh": return littleEndian{} case "armbe", "arm64be", + "m68k", "mips", "mips64", "mips64p32", "ppc", "ppc64", "s390", "s390x", + "shbe", "sparc", "sparc64": return bigEndian{} } diff --git a/vendor/golang.org/x/sys/windows/memory_windows.go b/vendor/golang.org/x/sys/windows/memory_windows.go index f80a4204f097d..e409d76f0fde7 100644 --- a/vendor/golang.org/x/sys/windows/memory_windows.go +++ b/vendor/golang.org/x/sys/windows/memory_windows.go @@ -23,4 +23,9 @@ const ( PAGE_EXECUTE_READ = 0x20 PAGE_EXECUTE_READWRITE = 0x40 PAGE_EXECUTE_WRITECOPY = 0x80 + + QUOTA_LIMITS_HARDWS_MIN_DISABLE = 0x00000002 + QUOTA_LIMITS_HARDWS_MIN_ENABLE = 0x00000001 + QUOTA_LIMITS_HARDWS_MAX_DISABLE = 0x00000008 + QUOTA_LIMITS_HARDWS_MAX_ENABLE = 0x00000004 ) diff --git a/vendor/golang.org/x/sys/windows/syscall_windows.go b/vendor/golang.org/x/sys/windows/syscall_windows.go index 12c0544cb5aeb..62cf70e9f6702 100644 --- a/vendor/golang.org/x/sys/windows/syscall_windows.go +++ b/vendor/golang.org/x/sys/windows/syscall_windows.go @@ -308,6 +308,8 @@ func NewCallbackCDecl(fn interface{}) uintptr { //sys GetProcessId(process Handle) (id uint32, err error) //sys OpenThread(desiredAccess uint32, inheritHandle bool, threadId uint32) (handle Handle, err error) //sys SetProcessPriorityBoost(process Handle, disable bool) (err error) = kernel32.SetProcessPriorityBoost +//sys GetProcessWorkingSetSizeEx(hProcess Handle, lpMinimumWorkingSetSize *uintptr, lpMaximumWorkingSetSize *uintptr, flags *uint32) +//sys SetProcessWorkingSetSizeEx(hProcess Handle, dwMinimumWorkingSetSize uintptr, dwMaximumWorkingSetSize uintptr, flags uint32) (err error) // Volume Management Functions //sys DefineDosDevice(flags uint32, deviceName *uint16, targetPath *uint16) (err error) = DefineDosDeviceW diff --git a/vendor/golang.org/x/sys/windows/zsyscall_windows.go b/vendor/golang.org/x/sys/windows/zsyscall_windows.go index 2aa4fa642a3d4..8a562feed0d7e 100644 --- a/vendor/golang.org/x/sys/windows/zsyscall_windows.go +++ b/vendor/golang.org/x/sys/windows/zsyscall_windows.go @@ -217,6 +217,8 @@ var ( procGetProcessId = modkernel32.NewProc("GetProcessId") procOpenThread = modkernel32.NewProc("OpenThread") procSetProcessPriorityBoost = modkernel32.NewProc("SetProcessPriorityBoost") + procGetProcessWorkingSetSizeEx = modkernel32.NewProc("GetProcessWorkingSetSizeEx") + procSetProcessWorkingSetSizeEx = modkernel32.NewProc("SetProcessWorkingSetSizeEx") procDefineDosDeviceW = modkernel32.NewProc("DefineDosDeviceW") procDeleteVolumeMountPointW = modkernel32.NewProc("DeleteVolumeMountPointW") procFindFirstVolumeW = modkernel32.NewProc("FindFirstVolumeW") @@ -2414,6 +2416,23 @@ func SetProcessPriorityBoost(process Handle, disable bool) (err error) { return } +func GetProcessWorkingSetSizeEx(hProcess Handle, lpMinimumWorkingSetSize *uintptr, lpMaximumWorkingSetSize *uintptr, flags *uint32) { + syscall.Syscall6(procGetProcessWorkingSetSizeEx.Addr(), 4, uintptr(hProcess), uintptr(unsafe.Pointer(lpMinimumWorkingSetSize)), uintptr(unsafe.Pointer(lpMaximumWorkingSetSize)), uintptr(unsafe.Pointer(flags)), 0, 0) + return +} + +func SetProcessWorkingSetSizeEx(hProcess Handle, dwMinimumWorkingSetSize uintptr, dwMaximumWorkingSetSize uintptr, flags uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procSetProcessWorkingSetSizeEx.Addr(), 4, uintptr(hProcess), uintptr(dwMinimumWorkingSetSize), uintptr(dwMaximumWorkingSetSize), uintptr(flags), 0, 0) + if r1 == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} + func DefineDosDevice(flags uint32, deviceName *uint16, targetPath *uint16) (err error) { r1, _, e1 := syscall.Syscall(procDefineDosDeviceW.Addr(), 3, uintptr(flags), uintptr(unsafe.Pointer(deviceName)), uintptr(unsafe.Pointer(targetPath))) if r1 == 0 { diff --git a/vendor/modules.txt b/vendor/modules.txt index 2f39999817150..9232c015cfdd1 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -45,6 +45,7 @@ github.com/BurntSushi/toml # github.com/Masterminds/squirrel v0.0.0-20161115235646-20f192218cf5 github.com/Masterminds/squirrel # github.com/Microsoft/go-winio v0.4.12 +## explicit github.com/Microsoft/go-winio # github.com/NYTimes/gziphandler v1.1.1 github.com/NYTimes/gziphandler @@ -116,12 +117,15 @@ github.com/aws/aws-sdk-go/service/sts/stsiface # github.com/beorn7/perks v1.0.1 github.com/beorn7/perks/quantile # github.com/blang/semver v3.5.1+incompatible +## explicit github.com/blang/semver # github.com/bmatcuk/doublestar v1.2.2 +## explicit github.com/bmatcuk/doublestar # github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b github.com/bradfitz/gomemcache/memcache # github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee +## explicit github.com/c2h5oh/datasize # github.com/cenkalti/backoff v2.2.1+incompatible github.com/cenkalti/backoff @@ -130,19 +134,23 @@ github.com/cespare/xxhash # github.com/cespare/xxhash/v2 v2.1.1 github.com/cespare/xxhash/v2 # github.com/containerd/containerd v1.3.2 +## explicit github.com/containerd/containerd/errdefs # github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448 +## explicit github.com/containerd/fifo # github.com/coreos/go-semver v0.3.0 github.com/coreos/go-semver/semver # github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf +## explicit github.com/coreos/go-systemd/activation github.com/coreos/go-systemd/internal/dlopen github.com/coreos/go-systemd/journal github.com/coreos/go-systemd/sdjournal # github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f github.com/coreos/pkg/capnslog -# github.com/cortexproject/cortex v1.1.1-0.20200616130854-34b45d1180c3 +# github.com/cortexproject/cortex v1.1.1-0.20200625134921-0ea2318b0174 +## explicit github.com/cortexproject/cortex/pkg/alertmanager github.com/cortexproject/cortex/pkg/alertmanager/alerts github.com/cortexproject/cortex/pkg/alertmanager/alerts/configdb @@ -225,12 +233,17 @@ github.com/cortexproject/cortex/pkg/util/tls github.com/cortexproject/cortex/pkg/util/validation github.com/cortexproject/cortex/tools/querytee # github.com/davecgh/go-spew v1.1.1 +## explicit github.com/davecgh/go-spew/spew # github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/dgrijalva/jwt-go +# github.com/digitalocean/godo v1.37.0 +github.com/digitalocean/godo # github.com/docker/distribution v2.7.1+incompatible +## explicit github.com/docker/distribution/registry/api/errcode # github.com/docker/docker v0.7.3-0.20190817195342-4760db040282 +## explicit github.com/docker/docker/api/types github.com/docker/docker/api/types/backend github.com/docker/docker/api/types/blkiodev @@ -270,30 +283,39 @@ github.com/docker/go-connections/nat github.com/docker/go-connections/sockets github.com/docker/go-connections/tlsconfig # github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82 +## explicit github.com/docker/go-metrics # github.com/docker/go-plugins-helpers v0.0.0-20181025120712-1e6269c305b8 +## explicit github.com/docker/go-plugins-helpers/sdk # github.com/docker/go-units v0.4.0 github.com/docker/go-units # github.com/dustin/go-humanize v1.0.0 +## explicit github.com/dustin/go-humanize # github.com/edsrzf/mmap-go v1.0.0 github.com/edsrzf/mmap-go # github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb github.com/facette/natsort # github.com/fatih/color v1.9.0 +## explicit github.com/fatih/color # github.com/fluent/fluent-bit-go v0.0.0-20190925192703-ea13c021720c +## explicit github.com/fluent/fluent-bit-go/output +# github.com/frankban/quicktest v1.7.2 +## explicit # github.com/fsnotify/fsnotify v1.4.7 github.com/fsnotify/fsnotify # github.com/fsouza/fake-gcs-server v1.7.0 github.com/fsouza/fake-gcs-server/fakestorage github.com/fsouza/fake-gcs-server/internal/backend # github.com/go-kit/kit v0.10.0 +## explicit github.com/go-kit/kit/log github.com/go-kit/kit/log/level # github.com/go-logfmt/logfmt v0.5.0 +## explicit github.com/go-logfmt/logfmt # github.com/go-logr/logr v0.1.0 github.com/go-logr/logr @@ -333,10 +355,12 @@ github.com/gocql/gocql/internal/lru github.com/gocql/gocql/internal/murmur github.com/gocql/gocql/internal/streams # github.com/gofrs/flock v0.7.1 +## explicit github.com/gofrs/flock # github.com/gogo/googleapis v1.1.0 github.com/gogo/googleapis/google/rpc # github.com/gogo/protobuf v1.3.1 +## explicit github.com/gogo/protobuf/gogoproto github.com/gogo/protobuf/io github.com/gogo/protobuf/jsonpb @@ -370,6 +394,7 @@ github.com/golang/protobuf/ptypes/empty github.com/golang/protobuf/ptypes/timestamp github.com/golang/protobuf/ptypes/wrappers # github.com/golang/snappy v0.0.1 +## explicit github.com/golang/snappy # github.com/gomodule/redigo v2.0.0+incompatible github.com/gomodule/redigo/internal @@ -382,6 +407,8 @@ github.com/google/go-cmp/cmp/internal/diff github.com/google/go-cmp/cmp/internal/flags github.com/google/go-cmp/cmp/internal/function github.com/google/go-cmp/cmp/internal/value +# github.com/google/go-querystring v1.0.0 +github.com/google/go-querystring/query # github.com/google/gofuzz v1.1.0 github.com/google/gofuzz # github.com/google/uuid v1.1.1 @@ -409,16 +436,20 @@ github.com/gophercloud/gophercloud/openstack/objectstorage/v1/objects github.com/gophercloud/gophercloud/openstack/utils github.com/gophercloud/gophercloud/pagination # github.com/gorilla/mux v1.7.3 +## explicit github.com/gorilla/mux # github.com/gorilla/websocket v1.4.0 +## explicit github.com/gorilla/websocket # github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 +## explicit github.com/grpc-ecosystem/go-grpc-middleware github.com/grpc-ecosystem/go-grpc-middleware/recovery github.com/grpc-ecosystem/go-grpc-middleware/tags github.com/grpc-ecosystem/go-grpc-middleware/tracing/opentracing github.com/grpc-ecosystem/go-grpc-middleware/util/metautils # github.com/grpc-ecosystem/go-grpc-prometheus v1.2.1-0.20191002090509-6af20e3a5340 +## explicit github.com/grpc-ecosystem/go-grpc-prometheus github.com/grpc-ecosystem/go-grpc-prometheus/packages/grpcstatus # github.com/grpc-ecosystem/grpc-gateway v1.14.6 @@ -426,6 +457,7 @@ github.com/grpc-ecosystem/grpc-gateway/internal github.com/grpc-ecosystem/grpc-gateway/runtime github.com/grpc-ecosystem/grpc-gateway/utilities # github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 +## explicit github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc # github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed github.com/hailocab/go-hostpool @@ -448,6 +480,7 @@ github.com/hashicorp/go-rootcerts # github.com/hashicorp/go-sockaddr v1.0.2 github.com/hashicorp/go-sockaddr # github.com/hashicorp/golang-lru v0.5.4 +## explicit github.com/hashicorp/golang-lru github.com/hashicorp/golang-lru/simplelru # github.com/hashicorp/memberlist v0.2.0 @@ -455,12 +488,14 @@ github.com/hashicorp/memberlist # github.com/hashicorp/serf v0.9.0 github.com/hashicorp/serf/coordinate # github.com/hpcloud/tail v1.0.0 => github.com/grafana/tail v0.0.0-20191024143944-0b54ddf21fe7 +## explicit github.com/hpcloud/tail github.com/hpcloud/tail/ratelimiter github.com/hpcloud/tail/util github.com/hpcloud/tail/watch github.com/hpcloud/tail/winfile # github.com/influxdata/go-syslog/v3 v3.0.1-0.20200510134747-836dce2cf6da +## explicit github.com/influxdata/go-syslog/v3 github.com/influxdata/go-syslog/v3/common github.com/influxdata/go-syslog/v3/nontransparent @@ -469,14 +504,17 @@ github.com/influxdata/go-syslog/v3/rfc5424 # github.com/jessevdk/go-flags v1.4.0 github.com/jessevdk/go-flags # github.com/jmespath/go-jmespath v0.3.0 +## explicit github.com/jmespath/go-jmespath # github.com/jonboulle/clockwork v0.1.0 github.com/jonboulle/clockwork # github.com/joncrlsn/dque v2.2.1-0.20200515025108-956d14155fa2+incompatible +## explicit github.com/joncrlsn/dque # github.com/jpillora/backoff v1.0.0 github.com/jpillora/backoff # github.com/json-iterator/go v1.1.9 +## explicit github.com/json-iterator/go # github.com/jstemmer/go-junit-report v0.9.1 github.com/jstemmer/go-junit-report @@ -485,6 +523,7 @@ github.com/jstemmer/go-junit-report/parser # github.com/julienschmidt/httprouter v1.3.0 github.com/julienschmidt/httprouter # github.com/klauspost/compress v1.9.5 +## explicit github.com/klauspost/compress/flate github.com/klauspost/compress/gzip # github.com/konsorten/go-windows-terminal-sequences v1.0.2 @@ -527,14 +566,17 @@ github.com/minio/sha256-simd # github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/go-homedir # github.com/mitchellh/mapstructure v1.2.2 +## explicit github.com/mitchellh/mapstructure # github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd github.com/modern-go/concurrent # github.com/modern-go/reflect2 v1.0.1 github.com/modern-go/reflect2 # github.com/morikuni/aec v1.0.0 +## explicit github.com/morikuni/aec # github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f +## explicit github.com/mwitkow/go-conntrack # github.com/ncw/swift v1.0.50 github.com/ncw/swift @@ -552,13 +594,16 @@ github.com/opentracing-contrib/go-grpc # github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9 github.com/opentracing-contrib/go-stdlib/nethttp # github.com/opentracing/opentracing-go v1.1.1-0.20200124165624-2876d2018785 +## explicit github.com/opentracing/opentracing-go github.com/opentracing/opentracing-go/ext github.com/opentracing/opentracing-go/log # github.com/pierrec/lz4 v2.5.3-0.20200429092203-e876bbd321b3+incompatible +## explicit github.com/pierrec/lz4 github.com/pierrec/lz4/internal/xxh32 # github.com/pkg/errors v0.9.1 +## explicit github.com/pkg/errors # github.com/pmezard/go-difflib v1.0.0 github.com/pmezard/go-difflib/difflib @@ -603,6 +648,7 @@ github.com/prometheus/alertmanager/template github.com/prometheus/alertmanager/types github.com/prometheus/alertmanager/ui # github.com/prometheus/client_golang v1.6.1-0.20200604110148-03575cad4e55 +## explicit github.com/prometheus/client_golang/api github.com/prometheus/client_golang/api/prometheus/v1 github.com/prometheus/client_golang/prometheus @@ -613,8 +659,10 @@ github.com/prometheus/client_golang/prometheus/push github.com/prometheus/client_golang/prometheus/testutil github.com/prometheus/client_golang/prometheus/testutil/promlint # github.com/prometheus/client_model v0.2.0 +## explicit github.com/prometheus/client_model/go # github.com/prometheus/common v0.10.0 +## explicit github.com/prometheus/common/config github.com/prometheus/common/expfmt github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg @@ -627,12 +675,14 @@ github.com/prometheus/node_exporter/https github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs github.com/prometheus/procfs/internal/util -# github.com/prometheus/prometheus v1.8.2-0.20200609052543-1627d234da06 +# github.com/prometheus/prometheus v1.8.2-0.20200622142935-153f859b7499 +## explicit github.com/prometheus/prometheus/config github.com/prometheus/prometheus/discovery github.com/prometheus/prometheus/discovery/azure github.com/prometheus/prometheus/discovery/config github.com/prometheus/prometheus/discovery/consul +github.com/prometheus/prometheus/discovery/digitalocean github.com/prometheus/prometheus/discovery/dns github.com/prometheus/prometheus/discovery/ec2 github.com/prometheus/prometheus/discovery/file @@ -696,10 +746,12 @@ github.com/segmentio/fasthash/fnv1a # github.com/sercand/kuberesolver v2.4.0+incompatible github.com/sercand/kuberesolver # github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 +## explicit github.com/shurcooL/httpfs/filter github.com/shurcooL/httpfs/union github.com/shurcooL/httpfs/vfsutil # github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd +## explicit github.com/shurcooL/vfsgen # github.com/sirupsen/logrus v1.5.0 github.com/sirupsen/logrus @@ -713,10 +765,11 @@ github.com/spf13/pflag # github.com/stretchr/objx v0.2.0 github.com/stretchr/objx # github.com/stretchr/testify v1.5.1 +## explicit github.com/stretchr/testify/assert github.com/stretchr/testify/mock github.com/stretchr/testify/require -# github.com/thanos-io/thanos v0.12.3-0.20200603113103-6d3e730e154d +# github.com/thanos-io/thanos v0.12.3-0.20200618165043-6c513e5f5c5f github.com/thanos-io/thanos/pkg/block github.com/thanos-io/thanos/pkg/block/indexheader github.com/thanos-io/thanos/pkg/block/metadata @@ -754,8 +807,10 @@ github.com/thanos-io/thanos/pkg/tracing # github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 github.com/tmc/grpc-websocket-proxy/wsproxy # github.com/tonistiigi/fifo v0.0.0-20190226154929-a9fb20d87448 +## explicit github.com/tonistiigi/fifo # github.com/uber/jaeger-client-go v2.23.1+incompatible +## explicit github.com/uber/jaeger-client-go github.com/uber/jaeger-client-go/config github.com/uber/jaeger-client-go/internal/baggage @@ -777,9 +832,12 @@ github.com/uber/jaeger-client-go/utils # github.com/uber/jaeger-lib v2.2.0+incompatible github.com/uber/jaeger-lib/metrics github.com/uber/jaeger-lib/metrics/prometheus +# github.com/ugorji/go v1.1.7 +## explicit # github.com/ugorji/go/codec v1.1.7 github.com/ugorji/go/codec # github.com/weaveworks/common v0.0.0-20200512154658-384f10054ec5 +## explicit github.com/weaveworks/common/aws github.com/weaveworks/common/errors github.com/weaveworks/common/grpc @@ -797,7 +855,8 @@ github.com/weaveworks/common/user github.com/weaveworks/promrus # github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 github.com/xiang90/probing -# go.etcd.io/bbolt v1.3.4 +# go.etcd.io/bbolt v1.3.5-0.20200615073812-232d8fc87f50 +## explicit go.etcd.io/bbolt # go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f go.etcd.io/etcd/auth @@ -929,6 +988,7 @@ golang.org/x/lint/golint golang.org/x/mod/module golang.org/x/mod/semver # golang.org/x/net v0.0.0-20200602114024-627f9648deb9 +## explicit golang.org/x/net/bpf golang.org/x/net/context golang.org/x/net/context/ctxhttp @@ -956,7 +1016,7 @@ golang.org/x/oauth2/jwt # golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a golang.org/x/sync/errgroup golang.org/x/sync/semaphore -# golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 +# golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 golang.org/x/sys/cpu golang.org/x/sys/internal/unsafeheader golang.org/x/sys/unix @@ -1039,6 +1099,7 @@ google.golang.org/genproto/googleapis/rpc/status google.golang.org/genproto/googleapis/type/expr google.golang.org/genproto/protobuf/field_mask # google.golang.org/grpc v1.29.1 +## explicit google.golang.org/grpc google.golang.org/grpc/attributes google.golang.org/grpc/backoff @@ -1138,8 +1199,10 @@ google.golang.org/protobuf/types/known/timestamppb google.golang.org/protobuf/types/known/wrapperspb google.golang.org/protobuf/types/pluginpb # gopkg.in/alecthomas/kingpin.v2 v2.2.6 +## explicit gopkg.in/alecthomas/kingpin.v2 # gopkg.in/fsnotify.v1 v1.4.7 +## explicit gopkg.in/fsnotify.v1 # gopkg.in/fsnotify/fsnotify.v1 v1.4.7 gopkg.in/fsnotify/fsnotify.v1 @@ -1150,6 +1213,7 @@ gopkg.in/ini.v1 # gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 gopkg.in/tomb.v1 # gopkg.in/yaml.v2 v2.3.0 +## explicit gopkg.in/yaml.v2 # gopkg.in/yaml.v3 v3.0.0-20200603094226-e3079894b1e8 gopkg.in/yaml.v3 @@ -1327,6 +1391,7 @@ k8s.io/client-go/util/flowcontrol k8s.io/client-go/util/keyutil k8s.io/client-go/util/workqueue # k8s.io/klog v1.0.0 +## explicit k8s.io/klog # k8s.io/klog/v2 v2.0.0 k8s.io/klog/v2 @@ -1341,3 +1406,9 @@ rsc.io/binaryregexp/syntax sigs.k8s.io/structured-merge-diff/v3/value # sigs.k8s.io/yaml v1.2.0 sigs.k8s.io/yaml +# github.com/hpcloud/tail => github.com/grafana/tail v0.0.0-20191024143944-0b54ddf21fe7 +# github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go v36.2.0+incompatible +# github.com/Azure/go-autorest => github.com/Azure/go-autorest v13.3.0+incompatible +# k8s.io/client-go => k8s.io/client-go v0.18.3 +# github.com/satori/go.uuid => github.com/satori/go.uuid v1.2.0 +# github.com/gocql/gocql => github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85