diff --git a/daemon/logger/context.go b/daemon/logger/context.go new file mode 100644 index 0000000000000..3526d563e8d18 --- /dev/null +++ b/daemon/logger/context.go @@ -0,0 +1,72 @@ +package logger + +import ( + "fmt" + "os" + "strings" + "time" +) + +// Context provides enough information for a logging driver to do its function. +type Context struct { + Config map[string]string + ContainerID string + ContainerName string + ContainerEntrypoint string + ContainerArgs []string + ContainerImageID string + ContainerImageName string + ContainerCreated time.Time + LogPath string +} + +// Hostname returns the hostname from the underlying OS. +func (ctx *Context) Hostname() (string, error) { + hostname, err := os.Hostname() + if err != nil { + return "", fmt.Errorf("logger: can not resolve hostname: %v", err) + } + return hostname, nil +} + +// Command returns the command that the container being logged was +// started with. The Entrypoint is prepended to the container +// arguments. +func (ctx *Context) Command() string { + terms := []string{ctx.ContainerEntrypoint} + for _, arg := range ctx.ContainerArgs { + terms = append(terms, arg) + } + command := strings.Join(terms, " ") + return command +} + +// ID Returns the Container ID shortened to 12 characters. +func (ctx *Context) ID() string { + return ctx.ContainerID[:12] +} + +// FullID is an alias of ContainerID. +func (ctx *Context) FullID() string { + return ctx.ContainerID +} + +// Name returns the ContainerName without a preceding '/'. +func (ctx *Context) Name() string { + return ctx.ContainerName[1:] +} + +// ImageID returns the ContainerImageID shortened to 12 characters. +func (ctx *Context) ImageID() string { + return ctx.ContainerImageID[:12] +} + +// ImageFullID is an alias of ContainerID. +func (ctx *Context) ImageFullID() string { + return ctx.ContainerImageID +} + +// ImageName is an alias of ContainerImageName +func (ctx *Context) ImageName() string { + return ctx.ContainerImageName +} diff --git a/daemon/logger/factory.go b/daemon/logger/factory.go index ffd03ea469050..5ec0f6731da2b 100644 --- a/daemon/logger/factory.go +++ b/daemon/logger/factory.go @@ -2,10 +2,7 @@ package logger import ( "fmt" - "os" - "strings" "sync" - "time" ) // Creator builds a logging driver instance with given context. @@ -15,40 +12,6 @@ type Creator func(Context) (Logger, error) // logging implementation. type LogOptValidator func(cfg map[string]string) error -// Context provides enough information for a logging driver to do its function. -type Context struct { - Config map[string]string - ContainerID string - ContainerName string - ContainerEntrypoint string - ContainerArgs []string - ContainerImageID string - ContainerImageName string - ContainerCreated time.Time - LogPath string -} - -// Hostname returns the hostname from the underlying OS. -func (ctx *Context) Hostname() (string, error) { - hostname, err := os.Hostname() - if err != nil { - return "", fmt.Errorf("logger: can not resolve hostname: %v", err) - } - return hostname, nil -} - -// Command returns the command that the container being logged was -// started with. The Entrypoint is prepended to the container -// arguments. -func (ctx *Context) Command() string { - terms := []string{ctx.ContainerEntrypoint} - for _, arg := range ctx.ContainerArgs { - terms = append(terms, arg) - } - command := strings.Join(terms, " ") - return command -} - type logdriverFactory struct { registry map[string]Creator optValidator map[string]LogOptValidator diff --git a/daemon/logger/fluentd/fluentd.go b/daemon/logger/fluentd/fluentd.go index d9b94a4ddd884..3ce0747473291 100644 --- a/daemon/logger/fluentd/fluentd.go +++ b/daemon/logger/fluentd/fluentd.go @@ -3,16 +3,15 @@ package fluentd import ( - "bytes" "fmt" "math" "net" "strconv" "strings" - "text/template" "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/logger" + "github.com/docker/docker/daemon/logger/loggerutils" "github.com/fluent/fluent-logger-golang/fluent" ) @@ -23,12 +22,6 @@ type fluentd struct { writer *fluent.Fluent } -type receiver struct { - ID string - FullID string - Name string -} - const ( name = "fluentd" defaultHostName = "localhost" @@ -48,10 +41,14 @@ func init() { func parseConfig(ctx logger.Context) (string, int, string, error) { host := defaultHostName port := defaultPort - tag := "docker." + ctx.ContainerID[:12] config := ctx.Config + tag, err := loggerutils.ParseLogTag(ctx, "docker.{{.ID}}") + if err != nil { + return "", 0, "", err + } + if address := config["fluentd-address"]; address != "" { if h, p, err := net.SplitHostPort(address); err != nil { if !strings.Contains(err.Error(), "missing port in address") { @@ -68,23 +65,6 @@ func parseConfig(ctx logger.Context) (string, int, string, error) { } } - if config["fluentd-tag"] != "" { - receiver := &receiver{ - ID: ctx.ContainerID[:12], - FullID: ctx.ContainerID, - Name: ctx.ContainerName, - } - tmpl, err := template.New("tag").Parse(config["fluentd-tag"]) - if err != nil { - return "", 0, "", err - } - buf := new(bytes.Buffer) - if err := tmpl.Execute(buf, receiver); err != nil { - return "", 0, "", err - } - tag = buf.String() - } - return host, port, tag, nil } @@ -130,6 +110,7 @@ func ValidateLogOpt(cfg map[string]string) error { switch key { case "fluentd-address": case "fluentd-tag": + case "tag": default: return fmt.Errorf("unknown log opt '%s' for fluentd log driver", key) } diff --git a/daemon/logger/gelf/gelf.go b/daemon/logger/gelf/gelf.go index 49fd3ff5a926e..e980a6ce94bae 100644 --- a/daemon/logger/gelf/gelf.go +++ b/daemon/logger/gelf/gelf.go @@ -14,6 +14,7 @@ import ( "github.com/Graylog2/go-gelf/gelf" "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/logger" + "github.com/docker/docker/daemon/logger/loggerutils" "github.com/docker/docker/pkg/urlutil" ) @@ -64,6 +65,12 @@ func New(ctx logger.Context) (logger.Logger, error) { // remove trailing slash from container name containerName := bytes.TrimLeft([]byte(ctx.ContainerName), "/") + // parse log tag + tag, err := loggerutils.ParseLogTag(ctx, "") + if err != nil { + return nil, err + } + fields := gelfFields{ hostname: hostname, containerID: ctx.ContainerID, @@ -71,7 +78,7 @@ func New(ctx logger.Context) (logger.Logger, error) { imageID: ctx.ContainerImageID, imageName: ctx.ContainerImageName, command: ctx.Command(), - tag: ctx.Config["gelf-tag"], + tag: tag, created: ctx.ContainerCreated, } @@ -135,6 +142,7 @@ func ValidateLogOpt(cfg map[string]string) error { switch key { case "gelf-address": case "gelf-tag": + case "tag": default: return fmt.Errorf("unknown log opt '%s' for gelf log driver", key) } diff --git a/daemon/logger/loggerutils/log_tag.go b/daemon/logger/loggerutils/log_tag.go new file mode 100644 index 0000000000000..df24379acf96a --- /dev/null +++ b/daemon/logger/loggerutils/log_tag.go @@ -0,0 +1,46 @@ +package loggerutils + +import ( + "bytes" + "fmt" + "text/template" + + "github.com/Sirupsen/logrus" + "github.com/docker/docker/daemon/logger" +) + +// ParseLogTag generates a context aware tag for consistency across different +// log drivers based on the context of the running container. +func ParseLogTag(ctx logger.Context, defaultTemplate string) (string, error) { + tagTemplate := lookupTagTemplate(ctx, defaultTemplate) + + tmpl, err := template.New("log-tag").Parse(tagTemplate) + if err != nil { + return "", err + } + buf := new(bytes.Buffer) + if err := tmpl.Execute(buf, &ctx); err != nil { + return "", err + } + + return buf.String(), nil +} + +func lookupTagTemplate(ctx logger.Context, defaultTemplate string) string { + tagTemplate := ctx.Config["tag"] + + deprecatedConfigs := []string{"syslog-tag", "gelf-tag", "fluentd-tag"} + for i := 0; tagTemplate == "" && i < len(deprecatedConfigs); i++ { + cfg := deprecatedConfigs[i] + if ctx.Config[cfg] != "" { + tagTemplate = ctx.Config[cfg] + logrus.Warn(fmt.Sprintf("Using log tag from deprecated log-opt '%s'. Please use: --log-opt tag=\"%s\"", cfg, tagTemplate)) + } + } + + if tagTemplate == "" { + tagTemplate = defaultTemplate + } + + return tagTemplate +} diff --git a/daemon/logger/loggerutils/log_tag_test.go b/daemon/logger/loggerutils/log_tag_test.go new file mode 100644 index 0000000000000..994508b161270 --- /dev/null +++ b/daemon/logger/loggerutils/log_tag_test.go @@ -0,0 +1,58 @@ +package loggerutils + +import ( + "testing" + + "github.com/docker/docker/daemon/logger" +) + +func TestParseLogTagDefaultTag(t *testing.T) { + ctx := buildContext(map[string]string{}) + tag, e := ParseLogTag(ctx, "{{.ID}}") + assertTag(t, e, tag, ctx.ID()) +} + +func TestParseLogTag(t *testing.T) { + ctx := buildContext(map[string]string{"tag": "{{.ImageName}}/{{.Name}}/{{.ID}}"}) + tag, e := ParseLogTag(ctx, "{{.ID}}") + assertTag(t, e, tag, "test-image/test-container/container-ab") +} + +func TestParseLogTagSyslogTag(t *testing.T) { + ctx := buildContext(map[string]string{"syslog-tag": "{{.ImageName}}/{{.Name}}/{{.ID}}"}) + tag, e := ParseLogTag(ctx, "{{.ID}}") + assertTag(t, e, tag, "test-image/test-container/container-ab") +} + +func TestParseLogTagGelfTag(t *testing.T) { + ctx := buildContext(map[string]string{"gelf-tag": "{{.ImageName}}/{{.Name}}/{{.ID}}"}) + tag, e := ParseLogTag(ctx, "{{.ID}}") + assertTag(t, e, tag, "test-image/test-container/container-ab") +} + +func TestParseLogTagFluentdTag(t *testing.T) { + ctx := buildContext(map[string]string{"fluentd-tag": "{{.ImageName}}/{{.Name}}/{{.ID}}"}) + tag, e := ParseLogTag(ctx, "{{.ID}}") + assertTag(t, e, tag, "test-image/test-container/container-ab") +} + +// Helpers + +func buildContext(cfg map[string]string) logger.Context { + return logger.Context{ + ContainerID: "container-abcdefghijklmnopqrstuvwxyz01234567890", + ContainerName: "/test-container", + ContainerImageID: "image-abcdefghijklmnopqrstuvwxyz01234567890", + ContainerImageName: "test-image", + Config: cfg, + } +} + +func assertTag(t *testing.T, e error, tag string, expected string) { + if e != nil { + t.Fatalf("Error generating tag: %q", e) + } + if tag != expected { + t.Fatalf("Wrong tag: %q, should be %q", tag, expected) + } +} diff --git a/daemon/logger/syslog/syslog.go b/daemon/logger/syslog/syslog.go index 2af99be385395..7d052a662fa1d 100644 --- a/daemon/logger/syslog/syslog.go +++ b/daemon/logger/syslog/syslog.go @@ -16,6 +16,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/logger" + "github.com/docker/docker/daemon/logger/loggerutils" "github.com/docker/docker/pkg/urlutil" ) @@ -61,9 +62,9 @@ func init() { // the context. Supported context configuration variables are // syslog-address, syslog-facility, & syslog-tag. func New(ctx logger.Context) (logger.Logger, error) { - tag := ctx.Config["syslog-tag"] - if tag == "" { - tag = ctx.ContainerID[:12] + tag, err := loggerutils.ParseLogTag(ctx, "{{.ID}}") + if err != nil { + return nil, err } proto, address, err := parseAddress(ctx.Config["syslog-address"]) @@ -146,6 +147,7 @@ func ValidateLogOpt(cfg map[string]string) error { case "syslog-address": case "syslog-facility": case "syslog-tag": + case "tag": default: return fmt.Errorf("unknown log opt '%s' for syslog log driver", key) } diff --git a/docs/misc/deprecated.md b/docs/misc/deprecated.md index d943491a18fa7..622f77a27cb31 100644 --- a/docs/misc/deprecated.md +++ b/docs/misc/deprecated.md @@ -12,13 +12,26 @@ parent = "mn_use_docker" The following list of features are deprecated. +### Driver Specific Log Tags +**Deprecated In Release: v1.9** + +**Target For Removal In Release: v1.11** + +Log tags are now generated in a standard way across different logging drivers. +Because of which, the driver specific log tag options `syslog-tag`, `gelf-tag` and +`fluentd-tag` have been deprecated in favor of the generic `tag` option. + + docker --log-driver=syslog --log-opt tag="{{.ImageName}}/{{.Name}}/{{.ID}}" + + + ### LXC built-in exec driver **Deprecated In Release: v1.8** **Target For Removal In Release: v1.10** The built-in LXC execution driver is deprecated for an external implementation. -The lxc-conf flag and API fields will also be removed. +The lxc-conf flag and API fields will also be removed. ### Old Command Line Options **Deprecated In Release: [v1.8.0](/release-notes/#docker-engine-1-8-0)** @@ -29,7 +42,7 @@ The flags `-d` and `--daemon` are deprecated in favor of the `daemon` subcommand docker daemon -H ... -The following single-dash (`-opt`) variant of certain command line options +The following single-dash (`-opt`) variant of certain command line options are deprecated and replaced with double-dash options (`--opt`): docker attach -nostdin diff --git a/docs/reference/logging/fluentd.md b/docs/reference/logging/fluentd.md index cecdce4a63110..044f2e9fa11d3 100644 --- a/docs/reference/logging/fluentd.md +++ b/docs/reference/logging/fluentd.md @@ -5,6 +5,7 @@ description = "Describes how to use the fluentd logging driver." keywords = ["Fluentd, docker, logging, driver"] [menu.main] parent = "smn_logging" +weight=2 +++ @@ -32,7 +33,7 @@ The `docker logs` command is not available for this logging driver. Some options are supported by specifying `--log-opt` as many times as needed: - `fluentd-address`: specify `host:port` to connect `localhost:24224` - - `fluentd-tag`: specify tag for fluentd message, which interpret some markup, ex `{{.ID}}`, `{{.FullID}}` or `{{.Name}}` `docker.{{.ID}}` + - `tag`: specify tag for fluentd message, which interpret some markup, ex `{{.ID}}`, `{{.FullID}}` or `{{.Name}}` `docker.{{.ID}}` Configure the default logging driver by passing the @@ -65,24 +66,12 @@ By default, the logging driver connects to `localhost:24224`. Supply the docker run --log-driver=fluentd --log-opt fluentd-address=myhost.local:24224 -### fluentd-tag +### tag -Every Fluentd's event has a tag that indicates where the log comes from. By -default, the driver uses the `docker.{{.ID}}` tag. Use the `fluentd-tag` option -to change this behavior. +By default, Docker uses the first 12 characters of the container ID to tag log messages. +Refer to the [log tag option documentation](/reference/logging/log_tags/) for customizing +the log tag format. -When specifying a `fluentd-tag` value, you can use the following markup tags: - - - `{{.ID}}`: short container id (12 characters) - - `{{.FullID}}`: full container id - - `{{.Name}}`: container name - -## Note regarding container names - -At startup time, the system sets the `container_name` field and `{{.Name}}` -in the tags to their values at startup. If you use `docker rename` to rename a -container, the new name is not be reflected in `fluentd` messages. Instead, -these messages continue to use the original container name. ## Fluentd daemon management with Docker diff --git a/docs/reference/logging/index.md b/docs/reference/logging/index.md index 7a066e019c206..27bac5730bc70 100644 --- a/docs/reference/logging/index.md +++ b/docs/reference/logging/index.md @@ -1,7 +1,7 @@ diff --git a/docs/reference/logging/log_tags.md b/docs/reference/logging/log_tags.md new file mode 100644 index 0000000000000..493b9a5b93848 --- /dev/null +++ b/docs/reference/logging/log_tags.md @@ -0,0 +1,50 @@ + + +# Log Tags + +The `tag` log option specifies how to format a tag that identifies the +container's log messages. By default, the system uses the first 12 characters of +the container id. To override this behavior, specify a `tag` option: + +``` +docker run --log-driver=fluentd --log-opt fluentd-address=myhost.local:24224 --log-opt tag="mailer" +``` + +Docker supports some special template markup you can use when specifying a tag's value: + +| Markup | Description | +|--------------------|------------------------------------------------------| +| `{{.ID}}` | The first 12 characters of the container id. | +| `{{.FullID}}` | The full container id. | +| `{{.Name}}` | The container name. | +| `{{.ImageID}}` | The first 12 characters of the container's image id. | +| `{{.ImageFullID}}` | The container's full image identifier. | +| `{{.ImageName}}` | The name of the image used by the container. | + +For example, specifying a `--log-opt tag="{{.ImageName}}/{{.Name}}/{{.ID}}"` value yields `syslog` log lines like: + +``` +Aug 7 18:33:19 HOSTNAME docker/hello-world/foobar/5790672ab6a0[9103]: Hello from Docker. +``` + +At startup time, the system sets the `container_name` field and `{{.Name}}` in +the tags. If you use `docker rename` to rename a container, the new name is not +reflected in the log messages. Instead, these messages continue to use the +original container name. + +For advanced usage, the generated tag's use [go +templates](http://golang.org/pkg/text/template/) and the container's [logging +context](https://github.com/docker/docker/blob/master/daemon/logger/context.go). + +>**Note**:The driver specific log options `syslog-tag`, `fluentd-tag` and +>`gelf-tag` still work for backwards compatibility. However, going forward you +>should standardize on using the generic `tag` log option instead. diff --git a/docs/reference/logging/overview.md b/docs/reference/logging/overview.md index b2147e12f1268..4e3f5bc7235a9 100644 --- a/docs/reference/logging/overview.md +++ b/docs/reference/logging/overview.md @@ -2,7 +2,7 @@ +++ title = "Configuring Logging Drivers" description = "Configure logging driver." -keywords = ["Fluentd, docker, logging, driver"] +keywords = ["docker, logging, driver, Fluentd"] [menu.main] parent = "smn_logging" weight=-1 @@ -25,9 +25,9 @@ container's logging driver. The following options are supported: | `fluentd` | Fluentd logging driver for Docker. Writes log messages to `fluentd` (forward input). | | `awslogs` | Amazon CloudWatch Logs logging driver for Docker. Writes log messages to Amazon CloudWatch Logs. | -The `docker logs`command is available only for the `json-file` logging driver. +The `docker logs`command is available only for the `json-file` logging driver. -### The json-file options +## json-file options The following logging options are supported for the `json-file` logging driver: @@ -39,16 +39,16 @@ Logs that reach `max-size` are rolled over. You can set the size in kilobytes(k) `max-file` specifies the maximum number of files that a log is rolled over before being discarded. eg `--log-opt max-file=100`. If `max-size` is not set, then `max-file` is not honored. -If `max-size` and `max-file` are set, `docker logs` only returns the log lines from the newest log file. +If `max-size` and `max-file` are set, `docker logs` only returns the log lines from the newest log file. -### The syslog options +## syslog options The following logging options are supported for the `syslog` logging driver: --log-opt syslog-address=[tcp|udp]://host:port --log-opt syslog-address=unix://path --log-opt syslog-facility=daemon - --log-opt syslog-tag="mailer" + --log-opt tag="mailer" `syslog-address` specifies the remote syslog server address where the driver connects to. If not specified it defaults to the local unix socket of the running system. @@ -83,22 +83,23 @@ the following named facilities: * `local6` * `local7` -The `syslog-tag` specifies a tag that identifies the container's syslog messages. By default, -the system uses the first 12 characters of the container id. To override this behavior, specify -a `syslog-tag` option +By default, Docker uses the first 12 characters of the container ID to tag log messages. +Refer to the [log tag option documentation](/reference/logging/log_tags/) for customizing +the log tag format. -## Specify journald options + +## journald options The `journald` logging driver stores the container id in the journal's `CONTAINER_ID` field. For detailed information on working with this logging driver, see [the journald logging driver](/reference/logging/journald/) reference documentation. -## Specify gelf options +## gelf options The GELF logging driver supports the following options: --log-opt gelf-address=udp://host:port - --log-opt gelf-tag="database" + --log-opt tag="database" The `gelf-address` option specifies the remote GELF server address that the driver connects to. Currently, only `udp` is supported as the transport and you must @@ -107,24 +108,21 @@ driver to a GELF remote server at `192.168.0.42` on port `12201` $ docker run --log-driver=gelf --log-opt gelf-address=udp://192.168.0.42:12201 -The `gelf-tag` option specifies a tag for easy container identification. +By default, Docker uses the first 12 characters of the container ID to tag log messages. +Refer to the [log tag option documentation](/reference/logging/log_tags/) for customizing +the log tag format. + -## Specify fluentd options +## fluentd options You can use the `--log-opt NAME=VALUE` flag to specify these additional Fluentd logging driver options. - `fluentd-address`: specify `host:port` to connect [localhost:24224] - - `fluentd-tag`: specify tag for `fluentd` message, - -When specifying a `fluentd-tag` value, you can use the following markup tags: - - - `{{.ID}}`: short container id (12 characters) - - `{{.FullID}}`: full container id - - `{{.Name}}`: container name + - `tag`: specify tag for `fluentd` message, For example, to specify both additional options: -`docker run --log-driver=fluentd --log-opt fluentd-address=localhost:24224 --log-opt fluentd-tag=docker.{{.Name}}` +`docker run --log-driver=fluentd --log-opt fluentd-address=localhost:24224 --log-opt tag=docker.{{.Name}}` If container cannot connect to the Fluentd daemon on the specified address, the container stops immediately. For detailed information on working with this