[Enhancement] Add RotateOnStartup feature flag for file output (elast…

…ic#19347) * add RotateOnStartup feature enhance file output to also be able to disable file rotation on startup, which is enabled by default (see rotator) * document change * add RotateOnStartup feature enhance file output to also be able to disable file rotation on startup, which is enabled by default (see rotator) * document change * add change again * add RotateOnStartup feature enhance file output to also be able to disable file rotation on startup, which is enabled by default (see rotator) * document change * add change again * add RotateOnStartup feature * update reference.yml files (make update) Co-authored-by: Claus Klammer <[email protected]>
narph · Feb 19, 2021 · 3319f5b · 3319f5b
1 parent a7f820f
commit 3319f5b
Show file tree

Hide file tree

Showing 18 changed files with 60 additions and 22 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -572,6 +572,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add keystore support for autodiscover static configurations. {pull]16306[16306]
 - Add TLS support to Kerberos authentication in Elasticsearch. {pull}18607[18607]
 - Add support for multiple sets of hints on autodiscover {pull}18883[18883]
+- Add config option `rotate_on_startup` to file output {issue}19150[19150] {pull}19347[19347]
 - Add a configurable delay between retries when an app metadata cannot be retrieved by `add_cloudfoundry_metadata`. {pull}19181[19181]
 - Added the `max_cached_sessions` option to the script processor. {pull}19562[19562]
 - Add support for DNS over TLS for the dns_processor. {pull}19321[19321]

diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml
@@ -1103,7 +1103,9 @@ output.elasticsearch:
 
   # Permissions to use for file creation. The default is 0600.
   #permissions: 0600
-
+
+  # Configure automatic file rotation on every startup. The default is true.
+  #rotate_on_startup: true
 # ------------------------------- Console Output -------------------------------
 #output.console:
   # Boolean flag to enable or disable the output module.

diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
@@ -1995,7 +1995,9 @@ output.elasticsearch:
 
   # Permissions to use for file creation. The default is 0600.
   #permissions: 0600
-
+
+  # Configure automatic file rotation on every startup. The default is true.
+  #rotate_on_startup: true
 # ------------------------------- Console Output -------------------------------
 #output.console:
   # Boolean flag to enable or disable the output module.

diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml
@@ -1281,7 +1281,9 @@ output.elasticsearch:
 
   # Permissions to use for file creation. The default is 0600.
   #permissions: 0600
-
+
+  # Configure automatic file rotation on every startup. The default is true.
+  #rotate_on_startup: true
 # ------------------------------- Console Output -------------------------------
 #output.console:
   # Boolean flag to enable or disable the output module.

diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml
@@ -1046,7 +1046,9 @@ output.elasticsearch:
 
   # Permissions to use for file creation. The default is 0600.
   #permissions: 0600
-
+
+  # Configure automatic file rotation on every startup. The default is true.
+  #rotate_on_startup: true
 # ------------------------------- Console Output -------------------------------
 #output.console:
   # Boolean flag to enable or disable the output module.

diff --git a/libbeat/_meta/config/output-file.reference.yml.tmpl b/libbeat/_meta/config/output-file.reference.yml.tmpl
@@ -31,3 +31,6 @@
 
   # Permissions to use for file creation. The default is 0600.
   #permissions: 0600
+
+  # Configure automatic file rotation on every startup. The default is true.
+  #rotate_on_startup: true
diff --git a/libbeat/outputs/fileout/config.go b/libbeat/outputs/fileout/config.go
@@ -25,19 +25,21 @@ import (
 )
 
 type config struct {
-	Path          string       `config:"path"`
-	Filename      string       `config:"filename"`
-	RotateEveryKb uint         `config:"rotate_every_kb" validate:"min=1"`
-	NumberOfFiles uint         `config:"number_of_files"`
-	Codec         codec.Config `config:"codec"`
-	Permissions   uint32       `config:"permissions"`
+	Path            string       `config:"path"`
+	Filename        string       `config:"filename"`
+	RotateEveryKb   uint         `config:"rotate_every_kb" validate:"min=1"`
+	NumberOfFiles   uint         `config:"number_of_files"`
+	Codec           codec.Config `config:"codec"`
+	Permissions     uint32       `config:"permissions"`
+	RotateOnStartup bool         `config:"rotate_on_startup"`
 }
 
 var (
 	defaultConfig = config{
-		NumberOfFiles: 7,
-		RotateEveryKb: 10 * 1024,
-		Permissions:   0600,
+		NumberOfFiles:   7,
+		RotateEveryKb:   10 * 1024,
+		Permissions:     0600,
+		RotateOnStartup: true,
 	}
 )
 

diff --git a/libbeat/outputs/fileout/docs/fileout.asciidoc b/libbeat/outputs/fileout/docs/fileout.asciidoc
@@ -22,6 +22,7 @@ output.file:
   #rotate_every_kb: 10000
   #number_of_files: 7
   #permissions: 0600
+  #rotate_on_startup: true
 ------------------------------------------------------------------------------
 
 ==== Configuration options
@@ -61,6 +62,10 @@ The number of files must be between 2 and 1024. The default is 7.
 
 Permissions to use for file creation. The default is 0600.
 
+===== `rotate_on_startup`
+
+If the output file already exists on startup, immediately rotate it and start writing to a new file instead of appending to the existing one. Defaults to true.
+
 ===== `codec`
 
 Output codec configuration. If the `codec` section is missing, events will be json encoded.

diff --git a/libbeat/outputs/fileout/file.go b/libbeat/outputs/fileout/file.go
@@ -87,6 +87,7 @@ func (out *fileOutput) init(beat beat.Info, c config) error {
 		file.MaxSizeBytes(c.RotateEveryKb*1024),
 		file.MaxBackups(c.NumberOfFiles),
 		file.Permissions(os.FileMode(c.Permissions)),
+		file.RotateOnStartup(c.RotateOnStartup),
 		file.WithLogger(logp.NewLogger("rotator").With(logp.Namespace("rotator"))),
 	)
 	if err != nil {

diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml
@@ -1892,7 +1892,9 @@ output.elasticsearch:
 
   # Permissions to use for file creation. The default is 0600.
   #permissions: 0600
-
+
+  # Configure automatic file rotation on every startup. The default is true.
+  #rotate_on_startup: true
 # ------------------------------- Console Output -------------------------------
 #output.console:
   # Boolean flag to enable or disable the output module.

diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml
@@ -1598,7 +1598,9 @@ output.elasticsearch:
 
   # Permissions to use for file creation. The default is 0600.
   #permissions: 0600
-
+
+  # Configure automatic file rotation on every startup. The default is true.
+  #rotate_on_startup: true
 # ------------------------------- Console Output -------------------------------
 #output.console:
   # Boolean flag to enable or disable the output module.

diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml
@@ -1026,7 +1026,9 @@ output.elasticsearch:
 
   # Permissions to use for file creation. The default is 0600.
   #permissions: 0600
-
+
+  # Configure automatic file rotation on every startup. The default is true.
+  #rotate_on_startup: true
 # ------------------------------- Console Output -------------------------------
 #output.console:
   # Boolean flag to enable or disable the output module.

diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml
@@ -1159,7 +1159,9 @@ output.elasticsearch:
 
   # Permissions to use for file creation. The default is 0600.
   #permissions: 0600
-
+
+  # Configure automatic file rotation on every startup. The default is true.
+  #rotate_on_startup: true
 # ------------------------------- Console Output -------------------------------
 #output.console:
   # Boolean flag to enable or disable the output module.

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -3908,7 +3908,9 @@ output.elasticsearch:
 
   # Permissions to use for file creation. The default is 0600.
   #permissions: 0600
-
+
+  # Configure automatic file rotation on every startup. The default is true.
+  #rotate_on_startup: true
 # ------------------------------- Console Output -------------------------------
 #output.console:
   # Boolean flag to enable or disable the output module.

diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml
@@ -1281,7 +1281,9 @@ output.elasticsearch:
 
   # Permissions to use for file creation. The default is 0600.
   #permissions: 0600
-
+
+  # Configure automatic file rotation on every startup. The default is true.
+  #rotate_on_startup: true
 # ------------------------------- Console Output -------------------------------
 #output.console:
   # Boolean flag to enable or disable the output module.

diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml
@@ -2393,7 +2393,9 @@ output.elasticsearch:
 
   # Permissions to use for file creation. The default is 0600.
   #permissions: 0600
-
+
+  # Configure automatic file rotation on every startup. The default is true.
+  #rotate_on_startup: true
 # ------------------------------- Console Output -------------------------------
 #output.console:
   # Boolean flag to enable or disable the output module.

diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml
@@ -1598,7 +1598,9 @@ output.elasticsearch:
 
   # Permissions to use for file creation. The default is 0600.
   #permissions: 0600
-
+
+  # Configure automatic file rotation on every startup. The default is true.
+  #rotate_on_startup: true
 # ------------------------------- Console Output -------------------------------
 #output.console:
   # Boolean flag to enable or disable the output module.

diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml
@@ -1069,7 +1069,9 @@ output.elasticsearch:
 
   # Permissions to use for file creation. The default is 0600.
   #permissions: 0600
-
+
+  # Configure automatic file rotation on every startup. The default is true.
+  #rotate_on_startup: true
 # ------------------------------- Console Output -------------------------------
 #output.console:
   # Boolean flag to enable or disable the output module.