[Metricbeat] Add tags for aws ec2 (elastic#12372)

* Add ec2 tags to aws.ec2.instance.tags.*
* Move tags under aws.tags.* a module field

CHANGELOG.next.asciidoc

@@ -231,6 +231,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add validation for elasticsearch and kibana modules' metricsets when xpack.enabled is set to true. {pull}12386[12386]
 - Add a system/entropy metricset {pull}12450[12450]
 - Allow redis URL format in redis hosts config. {pull}12408[12408]
+- Add tags into ec2 metricset. {issue}[12263]12263 {pull}12372[12372]
 
 *Packetbeat*
 

metricbeat/docs/fields.asciidoc

@@ -830,6 +830,16 @@ Total.
 
 
 
+*`aws.tags.*`*::
++
+--
+type: object
+
+Tag key value pairs from aws resources.
+
+
+--
+
 [float]
 == cloudwatch fields
 

metricbeat/docs/modules/aws.asciidoc

@@ -76,7 +76,10 @@ metricbeat.modules:
 [float]
 == Metricsets
 
-The following Metricsets are already included:
+Currently, we have `ec2`, `sqs`, `s3_request`, `s3_daily_storage` and `cloudwatch` metricset in `aws` module. Collecting `tags`
+for EC2 instances is supported.
+
+* *tags.*: Tag key value pairs from aws resources. A tag is a label that user assigns to an AWS resource.
 
 [float]
 === `ec2`

x-pack/metricbeat/module/aws/_meta/docs.asciidoc

@@ -69,7 +69,10 @@ metricbeat.modules:
 [float]
 == Metricsets
 
-The following Metricsets are already included:
+Currently, we have `ec2`, `sqs`, `s3_request`, `s3_daily_storage` and `cloudwatch` metricset in `aws` module. Collecting `tags`
+for EC2 instances is supported.
+
+* *tags.*: Tag key value pairs from aws resources. A tag is a label that user assigns to an AWS resource.
 
 [float]
 === `ec2`

x-pack/metricbeat/module/aws/_meta/fields.yml

@@ -8,3 +8,9 @@
       type: group
       description: >
       fields:
+        - name: tags.*
+          type: object
+          object_type: keyword
+          object_type_mapping_type: "*"
+          description: >
+            Tag key value pairs from aws resources.

x-pack/metricbeat/module/aws/aws.go

@@ -128,6 +128,7 @@ func InitEvent(metricsetName string, regionName string) mb.Event {
 	event := mb.Event{}
 	event.Service = metricsetName
 	event.MetricSetFields = common.MapStr{}
+	event.ModuleFields = common.MapStr{}
 	event.RootFields = common.MapStr{}
 	event.RootFields.Put("service.name", metricsetName)
 	event.RootFields.Put("cloud.provider", "aws")

x-pack/metricbeat/module/aws/ec2/_meta/data.json

@@ -3,12 +3,8 @@
     "aws": {
         "ec2": {
             "cpu": {
-                "credit_balance": 144,
-                "credit_usage": 0.005318,
-                "surplus_credit_balance": 0,
-                "surplus_credits_charged": 0,
                 "total": {
-                    "pct": 0.133333333333212
+                    "pct": 0.7000000000116418
                 }
             },
             "diskio": {
@@ -26,52 +22,55 @@
                     "count": 1
                 },
                 "image": {
-                    "id": "ami-05b3bcf7f311194b3"
+                    "id": "ami-62d35c02"
                 },
                 "monitoring": {
-                    "state": "disabled"
+                    "state": "enabled"
                 },
                 "private": {
-                    "dns_name": "ip-172-31-10-23.ap-southeast-1.compute.internal",
-                    "ip": "172.31.10.23"
+                    "dns_name": "ip-10-240-32-59.us-west-2.compute.internal",
+                    "ip": "10.240.32.59"
                 },
                 "public": {
-                    "dns_name": "ec2-18-136-198-93.ap-southeast-1.compute.amazonaws.com",
-                    "ip": "18.136.198.93"
+                    "dns_name": ""
                 },
                 "state": {
                     "code": 16,
                     "name": "running"
                 },
-                "threads_per_core": 1
+                "threads_per_core": 2
             },
             "network": {
                 "in": {
-                    "bytes": 583,
-                    "packets": 4.8
+                    "bytes": 16598.6,
+                    "packets": 127.8
                 },
                 "out": {
-                    "bytes": 746.4,
-                    "packets": 5
+                    "bytes": 29934.4,
+                    "packets": 225.4
                 }
             },
             "status": {
                 "check_failed": 0,
                 "check_failed_instance": 0,
                 "check_failed_system": 0
             }
+        },
+        "tags": {
+            "Name": "service-node",
+            "aws:autoscaling:groupName": "service-asg"
         }
     },
     "cloud": {
-        "availability_zone": "ap-southeast-1b",
+        "availability_zone": "us-west-2b",
         "instance": {
-            "id": "i-0c68eeb552231a8d0"
+            "id": "i-05cb8b44489d276f4"
         },
         "machine": {
-            "type": "t2.micro"
+            "type": "c4.large"
         },
         "provider": "aws",
-        "region": "ap-southeast-1"
+        "region": "us-west-2"
     },
     "event": {
         "dataset": "aws.ec2",
@@ -85,4 +84,4 @@
         "name": "ec2",
         "type": "ec2"
     }
-}
+}

x-pack/metricbeat/module/aws/ec2/ec2.go

@@ -210,8 +210,12 @@ func (m *MetricSet) createCloudWatchEvents(getMetricDataResults []cloudwatch.Met
 					events[instanceID].MetricSetFields.Put("instance.private.ip", *privateIP)
 				}
 
+				// Add tags
+				tags := instanceOutput[instanceID].Tags
+				for _, tag := range tags {
+					events[instanceID].ModuleFields.Put("tags."+*tag.Key, *tag.Value)
+				}
 			}
-
 		}
 	}