forked from moby/moby
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request moby#18766 from mikedanese/gcplogs
Add logging driver for Google Cloud Logging
- Loading branch information
Showing
111 changed files
with
26,604 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,181 @@ | ||
| package gcplogs | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "sync/atomic" | ||
| "time" | ||
|
|
||
| "github.com/docker/docker/daemon/logger" | ||
|
|
||
| "github.com/Sirupsen/logrus" | ||
| "golang.org/x/net/context" | ||
| "google.golang.org/cloud/compute/metadata" | ||
| "google.golang.org/cloud/logging" | ||
| ) | ||
|
|
||
| const ( | ||
| name = "gcplogs" | ||
|
|
||
| projectOptKey = "gcp-project" | ||
| logLabelsKey = "labels" | ||
| logEnvKey = "env" | ||
| logCmdKey = "gcp-log-cmd" | ||
| ) | ||
|
|
||
| var ( | ||
| // The number of logs the gcplogs driver has dropped. | ||
| droppedLogs uint64 | ||
|
|
||
| onGCE = metadata.OnGCE() | ||
|
|
||
| // instance metadata populated from the metadata server if available | ||
| projectID string | ||
| zone string | ||
| instanceName string | ||
| instanceID string | ||
| ) | ||
|
|
||
| func init() { | ||
| if onGCE { | ||
| // These will fail on instances if the metadata service is | ||
| // down or the client is compiled with an API version that | ||
| // has been removed. Since these are not vital, let's ignore | ||
| // them and make their fields in the dockeLogEntry ,omitempty | ||
| projectID, _ = metadata.ProjectID() | ||
| zone, _ = metadata.Zone() | ||
| instanceName, _ = metadata.InstanceName() | ||
| instanceID, _ = metadata.InstanceID() | ||
| } | ||
|
|
||
| if err := logger.RegisterLogDriver(name, New); err != nil { | ||
| logrus.Fatal(err) | ||
| } | ||
|
|
||
| if err := logger.RegisterLogOptValidator(name, ValidateLogOpts); err != nil { | ||
| logrus.Fatal(err) | ||
| } | ||
| } | ||
|
|
||
| type gcplogs struct { | ||
| client *logging.Client | ||
| instance *instanceInfo | ||
| container *containerInfo | ||
| } | ||
|
|
||
| type dockerLogEntry struct { | ||
| Instance *instanceInfo `json:"instance,omitempty"` | ||
| Container *containerInfo `json:"container,omitempty"` | ||
| Data string `json:"data,omitempty"` | ||
| } | ||
|
|
||
| type instanceInfo struct { | ||
| Zone string `json:"zone,omitempty"` | ||
| Name string `json:"name,omitempty"` | ||
| ID string `json:"id,omitempty"` | ||
| } | ||
|
|
||
| type containerInfo struct { | ||
| Name string `json:"name,omitempty"` | ||
| ID string `json:"id,omitempty"` | ||
| ImageName string `json:"imageName,omitempty"` | ||
| ImageID string `json:"imageId,omitempty"` | ||
| Created time.Time `json:"created,omitempty"` | ||
| Command string `json:"command,omitempty"` | ||
| Metadata map[string]string `json:"metadata,omitempty"` | ||
| } | ||
|
|
||
| // New creates a new logger that logs to Google Cloud Logging using the application | ||
| // default credentials. | ||
| // | ||
| // See https://developers.google.com/identity/protocols/application-default-credentials | ||
| func New(ctx logger.Context) (logger.Logger, error) { | ||
|
|
||
| var project string | ||
| if projectID != "" { | ||
| project = projectID | ||
| } | ||
| if projectID, found := ctx.Config[projectOptKey]; found { | ||
| project = projectID | ||
| } | ||
| if project == "" { | ||
| return nil, fmt.Errorf("No project was specified and couldn't read project from the meatadata server. Please specify a project") | ||
| } | ||
|
|
||
| c, err := logging.NewClient(context.Background(), project, "gcplogs-docker-driver") | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| if err := c.Ping(); err != nil { | ||
| return nil, fmt.Errorf("unable to connect or authenticate with Google Cloud Logging: %v", err) | ||
| } | ||
|
|
||
| l := &gcplogs{ | ||
| client: c, | ||
| container: &containerInfo{ | ||
| Name: ctx.ContainerName, | ||
| ID: ctx.ContainerID, | ||
| ImageName: ctx.ContainerImageName, | ||
| ImageID: ctx.ContainerImageID, | ||
| Created: ctx.ContainerCreated, | ||
| Metadata: ctx.ExtraAttributes(nil), | ||
| }, | ||
| } | ||
|
|
||
| if ctx.Config[logCmdKey] == "true" { | ||
| l.container.Command = ctx.Command() | ||
| } | ||
|
|
||
| if onGCE { | ||
| l.instance = &instanceInfo{ | ||
| Zone: zone, | ||
| Name: instanceName, | ||
| ID: instanceID, | ||
| } | ||
| } | ||
|
|
||
| // The logger "overflows" at a rate of 10,000 logs per second and this | ||
| // overflow func is called. We want to surface the error to the user | ||
| // without overly spamming /var/log/docker.log so we log the first time | ||
| // we overflow and every 1000th time after. | ||
| c.Overflow = func(_ *logging.Client, _ logging.Entry) error { | ||
| if i := atomic.AddUint64(&droppedLogs, 1); i%1000 == 1 { | ||
| logrus.Errorf("gcplogs driver has dropped %v logs", i) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| return l, nil | ||
| } | ||
|
|
||
| // ValidateLogOpts validates the opts passed to the gcplogs driver. Currently, the gcplogs | ||
| // driver doesn't take any arguments. | ||
| func ValidateLogOpts(cfg map[string]string) error { | ||
| for k := range cfg { | ||
| switch k { | ||
| case projectOptKey, logLabelsKey, logEnvKey, logCmdKey: | ||
| default: | ||
| return fmt.Errorf("%q is not a valid option for the gcplogs driver", k) | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func (l *gcplogs) Log(m *logger.Message) error { | ||
| return l.client.Log(logging.Entry{ | ||
| Time: m.Timestamp, | ||
| Payload: &dockerLogEntry{ | ||
| Instance: l.instance, | ||
| Container: l.container, | ||
| Data: string(m.Line), | ||
| }, | ||
| }) | ||
| } | ||
|
|
||
| func (l *gcplogs) Close() error { | ||
| return l.client.Flush() | ||
| } | ||
|
|
||
| func (l *gcplogs) Name() string { | ||
| return name | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| <!--[metadata]> | ||
| +++ | ||
| title = "Google Cloud Logging driver" | ||
| description = "Describes how to use the Google Cloud Logging driver." | ||
| keywords = ["gcplogs, google, docker, logging, driver"] | ||
| [menu.main] | ||
| parent = "smn_logging" | ||
| weight = 2 | ||
| +++ | ||
| <![end-metadata]--> | ||
|
|
||
| # Google Cloud Logging driver | ||
|
|
||
| The Google Cloud Logging driver sends container logs to <a href="https://cloud.google.com/logging/docs/" target="_blank">Google Cloud | ||
| Logging</a>. | ||
|
|
||
| ## Usage | ||
|
|
||
| You can configure the default logging driver by passing the `--log-driver` | ||
| option to the Docker daemon: | ||
|
|
||
| docker daemon --log-driver=gcplogs | ||
|
|
||
| You can set the logging driver for a specific container by using the | ||
| `--log-driver` option to `docker run`: | ||
|
|
||
| docker run --log-driver=gcplogs ... | ||
|
|
||
| This log driver does not implement a reader so it is incompatible with | ||
| `docker logs`. | ||
|
|
||
| If Docker detects that it is running in a Google Cloud Project, it will discover configuration | ||
| from the <a href="https://cloud.google.com/compute/docs/metadata" target="_blank">instance metadata service</a>. | ||
| Otherwise, the user must specify which project to log to using the `--gcp-project` | ||
| log option and Docker will attempt to obtain credentials from the | ||
| <a href="https://developers.google.com/identity/protocols/application-default-credentials" target="_blank">Google Application Default Credential</a>. | ||
| The `--gcp-project` takes precedence over information discovered from the metadata server | ||
| so a Docker daemon running in a Google Cloud Project can be overriden to log to a different | ||
| Google Cloud Project using `--gcp-project`. | ||
|
|
||
| ## gcplogs options | ||
|
|
||
| You can use the `--log-opt NAME=VALUE` flag to specify these additional Google | ||
| Cloud Logging driver options: | ||
|
|
||
| | Option | Required | Description | | ||
| |-----------------------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------| | ||
| | `gcp-project` | optional | Which GCP project to log to. Defaults to discovering this value from the GCE metadata service. | | ||
| | `gcp-log-cmd` | optional | Whether to log the command that the container was started with. Defaults to false. | | ||
| | `labels` | optional | Comma-separated list of keys of labels, which should be included in message, if these labels are specified for container. | | ||
| | `env` | optional | Comma-separated list of keys of environment variables, which should be included in message, if these variables are specified for container. | | ||
|
|
||
| If there is collision between `label` and `env` keys, the value of the `env` | ||
| takes precedence. Both options add additional fields to the attributes of a | ||
| logging message. | ||
|
|
||
| Below is an example of the logging options required to log to the default | ||
| logging destination which is discovered by querying the GCE metadata server. | ||
|
|
||
| docker run --log-driver=gcplogs \ | ||
| --log-opt labels=location | ||
| --log-opt env=TEST | ||
| --log-opt gcp-log-cmd=true | ||
| --env "TEST=false" | ||
| --label location=west | ||
| your/application | ||
|
|
||
| This configuration also directs the driver to include in the payload the label | ||
| `location`, the environment variable `ENV`, and the command used to start the | ||
| container. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| // Copyright 2015 The Go Authors. All rights reserved. | ||
| // Use of this source code is governed by a BSD-style | ||
| // license that can be found in the LICENSE file. | ||
|
|
||
| // +build go1.5 | ||
|
|
||
| package ctxhttp | ||
|
|
||
| import "net/http" | ||
|
|
||
| func canceler(client *http.Client, req *http.Request) func() { | ||
| ch := make(chan struct{}) | ||
| req.Cancel = ch | ||
|
|
||
| return func() { | ||
| close(ch) | ||
| } | ||
| } |
Oops, something went wrong.