Update metricbeat to use mage for all scenarios along with Makefile s…

…him (elastic#17799)

* Update metricbeat to use mage and the mage make targets.

* Run update and fmt.

* Remove assets.go as its no longer used.

* Split apart the metricbeat tests in travis into multiple jobs to take less time and not run over the timeline.

* Add back in create-metricset with mage.

* Re-add test_xpack_base.py.

* Add kafka to requirements.txt.

* Fix kafka to work with python 3.7.

* Remove the extra kafka-python from rebase on master.

* Fix issues from code review.

* Add back in comment from merge.

* Fix comment spacing.

* Fix update target in generate metricbeat.

* Update magefile in metricbeat generator.

* Run fmt and update.

.travis.yml

@@ -111,23 +111,22 @@ jobs:
     # Metricbeat
     - os: linux
       before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
-      env: TARGETS="-C metricbeat unit-tests coverage-report"
+      env: TARGETS="-C metricbeat unit-tests"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
-      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat vendor || travis_terminate 0
-      env: TARGETS="-C metricbeat integration-tests-environment coverage-report"
+      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C metricbeat integration-tests"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
-      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat vendor || travis_terminate 0
-      env: TARGETS="-C metricbeat update system-tests-environment coverage-report"
+      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C metricbeat system-tests"
       go: $TRAVIS_GO_VERSION
       stage: test
-
     - os: osx
       before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
-      env: TARGETS="TEST_ENVIRONMENT=0 -C metricbeat testsuite"
+      env: TARGETS="-C metricbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
@@ -137,6 +136,21 @@ jobs:
       stage: test
     - os: linux
       before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C x-pack/metricbeat unit-tests"
+      go: $TRAVIS_GO_VERSION
+      stage: test
+    - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C x-pack/metricbeat integration-tests"
+      go: $TRAVIS_GO_VERSION
+      stage: test
+    - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C x-pack/metricbeat system-tests"
+      go: $TRAVIS_GO_VERSION
+      stage: test
+    - os: osx
+      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
       env: TARGETS="-C x-pack/metricbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test

Jenkinsfile

@@ -319,7 +319,20 @@ pipeline {
             }
           }
           steps {
-            mageTarget("Metricbeat OSS linux/amd64 (integTest)", "metricbeat", "integTest")
+            mageTarget("Metricbeat OSS linux/amd64 (goIntegTest)", "metricbeat", "goIntegTest")
+          }
+        }
+        stage('Metricbeat Python integration tests'){
+          agent { label 'ubuntu && immutable' }
+          options { skipDefaultCheckout() }
+          when {
+            beforeAgent true
+            expression {
+              return env.BUILD_METRICBEAT != "false"
+            }
+          }
+          steps {
+            mageTarget("Metricbeat OSS linux/amd64 (pythonIntegTest)", "metricbeat", "pythonIntegTest")
           }
         }
         stage('Metricbeat x-pack'){

Makefile

@@ -28,7 +28,7 @@ PROJECTS_XPACK_MAGE=$(PROJECTS_XPACK_PKG) x-pack/libbeat
 #
 # Includes
 #
-include dev-tools/make/mage.mk
+include dev-tools/make/mage-install.mk
 
 ## help : Show this help.
 help: Makefile

dev-tools/make/mage-install.mk

@@ -0,0 +1,13 @@
+MAGE_VERSION     ?= v1.9.0
+MAGE_PRESENT     := $(shell mage --version 2> /dev/null | grep $(MAGE_VERSION))
+MAGE_IMPORT_PATH ?= github.com/magefile/mage
+export MAGE_IMPORT_PATH
+
+.PHONY: mage
+mage:
+ifndef MAGE_PRESENT
+	@echo Installing mage $(MAGE_VERSION) from vendor dir.
+	@go install -mod=vendor -ldflags="-X $(MAGE_IMPORT_PATH)/mage.gitTag=$(MAGE_VERSION)" ${MAGE_IMPORT_PATH}/...
+	@-mage -clean
+endif
+	@true

dev-tools/make/mage.mk

@@ -1,13 +1,69 @@
-MAGE_VERSION     ?= v1.9.0
-MAGE_PRESENT     := $(shell mage --version 2> /dev/null | grep $(MAGE_VERSION))
-MAGE_IMPORT_PATH ?= github.com/magefile/mage
-export MAGE_IMPORT_PATH
-
-.PHONY: mage
-mage:
-ifndef MAGE_PRESENT
-	@echo Installing mage $(MAGE_VERSION) from vendor dir.
-	@go install -mod=vendor -ldflags="-X $(MAGE_IMPORT_PATH)/mage.gitTag=$(MAGE_VERSION)" ${MAGE_IMPORT_PATH}/...
-	@-mage -clean
-endif
-	@true
+# This is a minimal Makefile for Beats that are built with Mage. Its only
+# responsibility is to provide compatibility with existing Jenkins and Travis
+# setups.
+
+#
+# Variables
+#
+.DEFAULT_GOAL := help
+PWD           := $(CURDIR)
+
+#
+# Includes
+#
+include $(ES_BEATS)/dev-tools/make/mage-install.mk
+
+#
+# Targets (alphabetically sorted).
+#
+.PHONY: check
+check: mage
+	mage check
+
+.PHONY: clean
+clean: mage
+	mage clean
+
+fix-permissions:
+
+.PHONY: fmt
+fmt: mage
+	mage fmt
+
+# Default target.
+.PHONY: help
+help:
+	@echo Use mage rather than make. Here are the available mage targets:
+	@mage -l
+
+.PHONY: release
+release: mage
+	mage package
+
+stop-environment:
+
+.PHONY: unit-tests
+unit-tests: mage
+	mage unitTest
+
+.PHONY: integration-tests
+integration-tests: mage
+	rm -f build/TEST-go-integration.out
+	mage goIntegTest || ( cat build/TEST-go-integration.out && false )
+
+.PHONY: system-tests
+system-tests: mage
+	mage pythonIntegTest
+
+.PHONY: testsuite
+testsuite: mage
+	rm -f build/TEST-go-integration.out
+	mage update build unitTest integTest || ( cat build/TEST-go-integration.out && false )
+
+.PHONY: update
+update: mage
+	mage update
+
+.PHONY: crosscompile
+crosscompile: mage
+	mage crossBuild

filebeat/docs/fields.asciidoc

@@ -28240,6 +28240,15 @@ type: keyword
 Palo Alto Networks name for the threat.
 
 
+type: keyword
+
+--
+
+*`panw.panos.action`*::
++
+--
+Action taken for the session.
+
 type: keyword
 
 --

generator/_templates/metricbeat/{beat}/magefile.go

@@ -14,7 +14,6 @@ import (
 	"github.com/elastic/beats/v7/dev-tools/mage/target/common"
 	"github.com/elastic/beats/v7/dev-tools/mage/target/pkg"
 	"github.com/elastic/beats/v7/dev-tools/mage/target/unittest"
-	"github.com/elastic/beats/v7/dev-tools/mage/target/update"
 	"github.com/elastic/beats/v7/generator/common/beatgen"
 	metricbeat "github.com/elastic/beats/v7/metricbeat/scripts/mage"
 )
@@ -45,7 +44,7 @@ func Package() {
 
 	devtools.UseCommunityBeatPackaging()
 
-	mg.Deps(update.Update)
+	mg.Deps(Update)
 	mg.Deps(build.CrossBuild, build.CrossBuildGoDaemon)
 	mg.SerialDeps(devtools.Package, pkg.PackageTest)
 }
@@ -99,9 +98,14 @@ func Fmt() {
 	common.Fmt()
 }
 
-// Update updates the generated files (aka make update).
-func Update() error {
-	return update.Update()
+// Update is an alias for running fields, dashboards, config.
+func Update() {
+	mg.SerialDeps(Fields, Dashboards, Config, Imports)
+}
+
+// Dashboards collects all the dashboards and generates index patterns.
+func Dashboards() error {
+	return devtools.KibanaDashboards("module")
 }
 
 // Imports generates an include/list.go file containing

generator/common/Makefile

@@ -5,7 +5,7 @@ BEAT_PATH=${GOPATH}/src/${BEAT_NAME}
 ES_BEATS=${GOPATH}/src/github.com/elastic/beats
 PREPARE_COMMAND?=
 
--include ${ES_BEATS}/dev-tools/make/mage.mk
+-include ${ES_BEATS}/dev-tools/make/mage-install.mk
 
 # Runs test build for mock beat
 .PHONY: test

libbeat/publisher/pipeline/testing.go

@@ -96,11 +96,14 @@ func (b *mockBatch) Events() []publisher.Event {
 	return b.events
 }
 
-func (b *mockBatch) ACK()                                 { signalFn(b.onACK) }
-func (b *mockBatch) Drop()                                { signalFn(b.onDrop) }
-func (b *mockBatch) Retry()                               { signalFn(b.onRetry) }
-func (b *mockBatch) Cancelled()                           { signalFn(b.onCancelled) }
-func (b *mockBatch) RetryEvents(events []publisher.Event) { b.updateEvents(events); signalFn(b.onRetry) }
+func (b *mockBatch) ACK()       { signalFn(b.onACK) }
+func (b *mockBatch) Drop()      { signalFn(b.onDrop) }
+func (b *mockBatch) Retry()     { signalFn(b.onRetry) }
+func (b *mockBatch) Cancelled() { signalFn(b.onCancelled) }
+func (b *mockBatch) RetryEvents(events []publisher.Event) {
+	b.updateEvents(events)
+	signalFn(b.onRetry)
+}
 
 func (b *mockBatch) reduceTTL() bool {
 	if b.onReduceTTL != nil {

libbeat/scripts/Makefile

@@ -128,7 +128,7 @@ endif
 #
 # Includes
 #
-include $(ES_BEATS)/dev-tools/make/mage.mk
+include $(ES_BEATS)/dev-tools/make/mage-install.mk
 
 ### BUILDING ###