Kubernetes metadata enhancements (elastic#22189)

Richardjgq · Nov 11, 2020 · 95a88a6 · 95a88a6
1 parent b9897e8
commit 95a88a6
Show file tree

Hide file tree

Showing 32 changed files with 286 additions and 82 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -523,6 +523,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for OpenStack SSL metadata APIs in `add_cloud_metadata`. {pull}21590[21590]
 - Add cloud.account.id for GCP into add_cloud_metadata processor. {pull}21776[21776]
 - Add proxy metricset for istio module. {pull}21751[21751]
+- Add kubernetes.node.hostname metadata of Kubernetes node. {pull}22189[22189]
+- Enable always add_resource_metadata for Pods and Services of kubernetes autodiscovery. {pull}22189[22189]
+- Add add_resource_metadata option setting (always enabled) for add_kubernetes_metadata setting. {pull}22189[22189]
 - Added Kafka version 2.2 to the list of supported versions. {pull}22328[22328]
 - Add support for ephemeral containers in kubernetes autodiscover and `add_kubernetes_metadata`. {pull}22389[22389] {pull}22439[22439]
 

diff --git a/auditbeat/docs/fields.asciidoc b/auditbeat/docs/fields.asciidoc
@@ -11460,6 +11460,16 @@ type: keyword
 Kubernetes node name
 
 
+type: keyword
+
+--
+
+*`kubernetes.node.hostname`*::
++
+--
+Kubernetes hostname as reported by the node’s kernel
+
+
 type: keyword
 
 --

diff --git a/auditbeat/include/fields.go b/auditbeat/include/fields.go
diff --git a/filebeat/docs/autodiscover-hints.asciidoc b/filebeat/docs/autodiscover-hints.asciidoc
@@ -172,7 +172,7 @@ The above configuration would generate two input configurations. The first input
 tokenizer. The second input handles everything but debug logs.
 
 [float]
-===== Configuring Namespace Defaults
+=====  Namespace Defaults
 
 Hints can be configured on the Namespace's annotations as defaults to use when Pod level annotations are missing.
 The resultant hints are a combination of Pod annotations and Namespace annotations with the Pod's taking precedence. To
@@ -186,7 +186,7 @@ filebeat.autodiscover:
       hints.enabled: true
       add_resource_metadata:
         namespace:
-          enabled: true
+          include_annotations: ["nsannotation1"]
 -------------------------------------------------------------------------------------
 
 

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -89343,6 +89343,16 @@ type: keyword
 Kubernetes node name
 
 
+type: keyword
+
+--
+
+*`kubernetes.node.hostname`*::
++
+--
+Kubernetes hostname as reported by the node’s kernel
+
+
 type: keyword
 
 --

diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go
diff --git a/heartbeat/docs/fields.asciidoc b/heartbeat/docs/fields.asciidoc
@@ -8957,6 +8957,16 @@ type: keyword
 Kubernetes node name
 
 
+type: keyword
+
+--
+
+*`kubernetes.node.hostname`*::
++
+--
+Kubernetes hostname as reported by the node’s kernel
+
+
 type: keyword
 
 --

diff --git a/heartbeat/include/fields.go b/heartbeat/include/fields.go
diff --git a/journalbeat/docs/fields.asciidoc b/journalbeat/docs/fields.asciidoc
@@ -9303,6 +9303,16 @@ type: keyword
 Kubernetes node name
 
 
+type: keyword
+
+--
+
+*`kubernetes.node.hostname`*::
++
+--
+Kubernetes hostname as reported by the node’s kernel
+
+
 type: keyword
 
 --

diff --git a/journalbeat/include/fields.go b/journalbeat/include/fields.go
diff --git a/libbeat/autodiscover/providers/kubernetes/node_test.go b/libbeat/autodiscover/providers/kubernetes/node_test.go
@@ -151,6 +151,10 @@ func TestEmitEvent_Node(t *testing.T) {
 							Type:    v1.NodeInternalIP,
 							Address: "1.2.3.4",
 						},
+						{
+							Type:    v1.NodeHostName,
+							Address: "node1",
+						},
 					},
 				},
 			},
@@ -161,16 +165,18 @@ func TestEmitEvent_Node(t *testing.T) {
 				"provider": UUID,
 				"kubernetes": common.MapStr{
 					"node": common.MapStr{
-						"name": "metricbeat",
-						"uid":  "005f3b90-4b9d-12f8-acf0-31020a840133",
+						"name":     "metricbeat",
+						"uid":      "005f3b90-4b9d-12f8-acf0-31020a840133",
+						"hostname": "node1",
 					},
 					"annotations": common.MapStr{},
 				},
 				"meta": common.MapStr{
 					"kubernetes": common.MapStr{
 						"node": common.MapStr{
-							"name": "metricbeat",
-							"uid":  "005f3b90-4b9d-12f8-acf0-31020a840133",
+							"name":     "metricbeat",
+							"uid":      "005f3b90-4b9d-12f8-acf0-31020a840133",
+							"hostname": "node1",
 						},
 					},
 				},
@@ -204,7 +210,7 @@ func TestEmitEvent_Node(t *testing.T) {
 				},
 				TypeMeta: typeMeta,
 				Status: v1.NodeStatus{
-					Addresses: []v1.NodeAddress{},
+					Addresses: []v1.NodeAddress{{Type: v1.NodeHostName, Address: "node1"}},
 					Conditions: []v1.NodeCondition{
 						{
 							Type:   v1.NodeReady,
@@ -220,16 +226,18 @@ func TestEmitEvent_Node(t *testing.T) {
 				"provider": UUID,
 				"kubernetes": common.MapStr{
 					"node": common.MapStr{
-						"name": "metricbeat",
-						"uid":  "005f3b90-4b9d-12f8-acf0-31020a840133",
+						"name":     "metricbeat",
+						"uid":      "005f3b90-4b9d-12f8-acf0-31020a840133",
+						"hostname": "node1",
 					},
 					"annotations": common.MapStr{},
 				},
 				"meta": common.MapStr{
 					"kubernetes": common.MapStr{
 						"node": common.MapStr{
-							"name": "metricbeat",
-							"uid":  "005f3b90-4b9d-12f8-acf0-31020a840133",
+							"name":     "metricbeat",
+							"uid":      "005f3b90-4b9d-12f8-acf0-31020a840133",
+							"hostname": "node1",
 						},
 					},
 				},

diff --git a/libbeat/autodiscover/providers/kubernetes/pod.go b/libbeat/autodiscover/providers/kubernetes/pod.go
@@ -75,43 +75,34 @@ func NewPodEventer(uuid uuid.UUID, cfg *common.Config, client k8s.Interface, pub
 		return nil, fmt.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Pod{}, err)
 	}
 
-	var nodeMeta, namespaceMeta metadata.MetaGen
-	var nodeWatcher, namespaceWatcher kubernetes.Watcher
+	options := kubernetes.WatchOptions{
+		SyncTimeout: config.SyncPeriod,
+		Node:        config.Node,
+	}
+	if config.Namespace != "" {
+		options.Namespace = config.Namespace
+	}
 	metaConf := config.AddResourceMetadata
-	if metaConf != nil {
-		if metaConf.Node != nil && metaConf.Node.Enabled() {
-			options := kubernetes.WatchOptions{
-				SyncTimeout: config.SyncPeriod,
-				Node:        config.Node,
-			}
-			if config.Namespace != "" {
-				options.Namespace = config.Namespace
-			}
-			nodeWatcher, err = kubernetes.NewWatcher(client, &kubernetes.Node{}, options, nil)
-			if err != nil {
-				return nil, fmt.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Node{}, err)
-			}
-
-			nodeMeta = metadata.NewNodeMetadataGenerator(metaConf.Node, nodeWatcher.Store())
-		}
-
-		if metaConf.Namespace != nil && metaConf.Namespace.Enabled() {
-			namespaceWatcher, err = kubernetes.NewWatcher(client, &kubernetes.Namespace{}, kubernetes.WatchOptions{
-				SyncTimeout: config.SyncPeriod,
-			}, nil)
-			if err != nil {
-				return nil, fmt.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Namespace{}, err)
-			}
-
-			namespaceMeta = metadata.NewNamespaceMetadataGenerator(metaConf.Namespace, namespaceWatcher.Store())
-		}
+	if metaConf == nil {
+		metaConf = metadata.GetDefaultResourceMetadataConfig()
+	}
+	nodeWatcher, err := kubernetes.NewWatcher(client, &kubernetes.Node{}, options, nil)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Node{}, err)
+	}
+	namespaceWatcher, err := kubernetes.NewWatcher(client, &kubernetes.Namespace{}, kubernetes.WatchOptions{
+		SyncTimeout: config.SyncPeriod,
+	}, nil)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Namespace{}, err)
 	}
+	metaGen := metadata.GetPodMetaGen(cfg, watcher, nodeWatcher, namespaceWatcher, metaConf)
 
 	p := &pod{
 		config:           config,
 		uuid:             uuid,
 		publish:          publish,
-		metagen:          metadata.NewPodMetadataGenerator(cfg, watcher.Store(), nodeMeta, namespaceMeta),
+		metagen:          metaGen,
 		logger:           logger,
 		watcher:          watcher,
 		nodeWatcher:      nodeWatcher,

diff --git a/libbeat/autodiscover/providers/kubernetes/service.go b/libbeat/autodiscover/providers/kubernetes/service.go
@@ -65,21 +65,18 @@ func NewServiceEventer(uuid uuid.UUID, cfg *common.Config, client k8s.Interface,
 
 	var namespaceMeta metadata.MetaGen
 	var namespaceWatcher kubernetes.Watcher
-	metaConf := config.AddResourceMetadata
-	if metaConf != nil {
-		if metaConf.Namespace != nil && metaConf.Namespace.Enabled() {
-			namespaceWatcher, err = kubernetes.NewWatcher(client, &kubernetes.Namespace{}, kubernetes.WatchOptions{
-				SyncTimeout: config.SyncPeriod,
-				Namespace:   config.Namespace,
-			}, nil)
-			if err != nil {
-				return nil, fmt.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Namespace{}, err)
-			}
 
-			namespaceMeta = metadata.NewNamespaceMetadataGenerator(metaConf.Namespace, namespaceWatcher.Store())
-		}
+	metaConf := metadata.GetDefaultResourceMetadataConfig()
+	namespaceWatcher, err = kubernetes.NewWatcher(client, &kubernetes.Namespace{}, kubernetes.WatchOptions{
+		SyncTimeout: config.SyncPeriod,
+		Namespace:   config.Namespace,
+	}, nil)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Namespace{}, err)
 	}
 
+	namespaceMeta = metadata.NewNamespaceMetadataGenerator(metaConf.Namespace, namespaceWatcher.Store())
+
 	p := &service{
 		config:           config,
 		uuid:             uuid,

diff --git a/libbeat/common/kubernetes/metadata/config.go b/libbeat/common/kubernetes/metadata/config.go
@@ -49,3 +49,13 @@ func (c *Config) InitDefaults() {
 func (c *Config) Unmarshal(cfg *common.Config) error {
 	return cfg.Unpack(c)
 }
+
+func GetDefaultResourceMetadataConfig() *AddResourceMetadataConfig {
+	metaConfig := Config{}
+	metaConfig.InitDefaults()
+	metaCfg, _ := common.NewConfigFrom(&metaConfig)
+	return &AddResourceMetadataConfig{
+		Node:      metaCfg,
+		Namespace: metaCfg,
+	}
+}
diff --git a/libbeat/common/kubernetes/metadata/metadata.go b/libbeat/common/kubernetes/metadata/metadata.go
@@ -50,3 +50,19 @@ func WithLabels(kind string) FieldOptions {
 		safemapstr.Put(meta, strings.ToLower(kind)+".labels", meta["labels"])
 	}
 }
+
+// GetPodMetaGen is a wrapper function that creates a metaGen for pod resource and has embeeded
+// nodeMetaGen and namespaceMetaGen
+func GetPodMetaGen(
+	cfg *common.Config,
+	podWatcher kubernetes.Watcher,
+	nodeWatcher kubernetes.Watcher,
+	namespaceWatcher kubernetes.Watcher,
+	metaConf *AddResourceMetadataConfig) MetaGen {
+
+	nodeMetaGen := NewNodeMetadataGenerator(metaConf.Node, nodeWatcher.Store())
+	namespaceMetaGen := NewNamespaceMetadataGenerator(metaConf.Namespace, namespaceWatcher.Store())
+	metaGen := NewPodMetadataGenerator(cfg, podWatcher.Store(), nodeMetaGen, namespaceMetaGen)
+
+	return metaGen
+}
diff --git a/libbeat/common/kubernetes/metadata/node.go b/libbeat/common/kubernetes/metadata/node.go
@@ -18,6 +18,7 @@
 package metadata
 
 import (
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/tools/cache"
 
 	"github.com/elastic/beats/v7/libbeat/common"
@@ -39,13 +40,17 @@ func NewNodeMetadataGenerator(cfg *common.Config, nodes cache.Store) MetaGen {
 
 // Generate generates service metadata from a resource object
 func (n *node) Generate(obj kubernetes.Resource, opts ...FieldOptions) common.MapStr {
-	_, ok := obj.(*kubernetes.Node)
+	node, ok := obj.(*kubernetes.Node)
 	if !ok {
 		return nil
 	}
 
 	meta := n.resource.Generate("node", obj, opts...)
 	// TODO: Add extra fields in here if need be
+	hostname := getHostName(node)
+	if hostname != "" {
+		meta.Put("node.hostname", getHostName(node))
+	}
 	return meta
 }
 
@@ -66,3 +71,13 @@ func (n *node) GenerateFromName(name string, opts ...FieldOptions) common.MapStr
 
 	return nil
 }
+
+// getHostName returns the HostName address of the node
+func getHostName(node *v1.Node) string {
+	for _, adr := range node.Status.Addresses {
+		if adr.Type == v1.NodeHostName {
+			return adr.Address
+		}
+	}
+	return ""
+}
diff --git a/libbeat/common/kubernetes/metadata/node_test.go b/libbeat/common/kubernetes/metadata/node_test.go
@@ -56,11 +56,15 @@ func TestNode_Generate(t *testing.T) {
 					Kind:       "Node",
 					APIVersion: "v1",
 				},
+				Status: v1.NodeStatus{
+					Addresses: []v1.NodeAddress{{Type: v1.NodeHostName, Address: "node1"}},
+				},
 			},
 			output: common.MapStr{
 				"node": common.MapStr{
-					"name": "obj",
-					"uid":  uid,
+					"name":     "obj",
+					"uid":      uid,
+					"hostname": "node1",
 				},
 				"labels": common.MapStr{
 					"foo": "bar",
@@ -101,11 +105,15 @@ func TestNode_GenerateFromName(t *testing.T) {
 					Kind:       "Node",
 					APIVersion: "v1",
 				},
+				Status: v1.NodeStatus{
+					Addresses: []v1.NodeAddress{{Type: v1.NodeHostName, Address: "node1"}},
+				},
 			},
 			output: common.MapStr{
 				"node": common.MapStr{
-					"name": "obj",
-					"uid":  uid,
+					"name":     "obj",
+					"uid":      uid,
+					"hostname": "node1",
 				},
 				"labels": common.MapStr{
 					"foo": "bar",

diff --git a/libbeat/common/kubernetes/metadata/pod_test.go b/libbeat/common/kubernetes/metadata/pod_test.go
@@ -314,6 +314,9 @@ func TestPod_GenerateWithNodeNamespace(t *testing.T) {
 					Kind:       "Node",
 					APIVersion: "v1",
 				},
+				Status: v1.NodeStatus{
+					Addresses: []v1.NodeAddress{{Type: v1.NodeHostName, Address: "node1"}},
+				},
 			},
 			namespace: &v1.Namespace{
 				ObjectMeta: metav1.ObjectMeta{
@@ -345,6 +348,7 @@ func TestPod_GenerateWithNodeNamespace(t *testing.T) {
 					"labels": common.MapStr{
 						"nodekey": "nodevalue",
 					},
+					"hostname": "node1",
 				},
 				"labels": common.MapStr{
 					"foo": "bar",