[tf] cloudtrail to cloudwatch to kinesis support (airbnb#750)

* [tf] adding support to send cloudtrail logs to cloudwatch

* [cli] updating cloudtrail generation code

* [tests] updating tests

* addressing pr feedback

stream_alert_cli/terraform/cloudtrail.py

@@ -36,6 +36,8 @@ def generate_cloudtrail(cluster_name, cluster_dict, config):
 
     cloudtrail_enabled = modules['cloudtrail'].get('enable_logging', True)
     kinesis_enabled = modules['cloudtrail'].get('enable_kinesis', True)
+    send_to_cloudwatch = modules['cloudtrail'].get('send_to_cloudwatch', False)
+    exclude_home_region = modules['cloudtrail'].get('exclude_home_region_events', False)
 
     account_ids = list(
         set([config['global']['account']['aws_account_id']] + modules['cloudtrail'].get(
@@ -53,6 +55,7 @@ def generate_cloudtrail(cluster_name, cluster_dict, config):
 
     existing_trail = modules['cloudtrail'].get('existing_trail', False)
     is_global_trail = modules['cloudtrail'].get('is_global_trail', True)
+    region = config['global']['account']['region']
 
     event_pattern_default = json.dumps({'account': [config['global']['account']['aws_account_id']]})
     try:
@@ -70,7 +73,7 @@ def generate_cloudtrail(cluster_name, cluster_dict, config):
         LOGGER_CLI.error('Config Error: Invalid CloudWatch Event Pattern!')
         return False
 
-    cluster_dict['module'][cloudtrail_module] = {
+    module_info = {
         'source': 'modules/tf_stream_alert_cloudtrail',
         'account_ids': account_ids,
         'cluster': cluster_name,
@@ -80,12 +83,24 @@ def generate_cloudtrail(cluster_name, cluster_dict, config):
         's3_logging_bucket':
         '{}.streamalert.s3-logging'.format(config['global']['account']['prefix']),
         'existing_trail': existing_trail,
+        'send_to_cloudwatch': send_to_cloudwatch,
+        'exclude_home_region_events': exclude_home_region,
+        'region': region,
         'is_global_trail': is_global_trail
     }
 
-    if kinesis_enabled:
-        cluster_dict['module'][cloudtrail_module][
-            'kinesis_arn'] = '${{module.kinesis_{}.arn}}'.format(cluster_name)
-        cluster_dict['module'][cloudtrail_module]['event_pattern'] = json.dumps(event_pattern)
+    # If kinesis or cloudwatch logs support is enabled, add some additional info
+    cloudwatch_enabled = modules.get('cloudwatch', {}).get('enabled')
+    if kinesis_enabled or cloudwatch_enabled:
+        # use the kinesis output from the kinesis streams module
+        module_info['kinesis_arn'] = '${{module.kinesis_{}.arn}}'.format(cluster_name)
+        if kinesis_enabled:
+            module_info['event_pattern'] = json.dumps(event_pattern)
+        else:
+            module_info['subscription_role_arn'] = ('${{module.cloudwatch_{}_{}.cloudwatch_'
+                                                    'subscription_role_arn}}'.format(cluster_name,
+                                                                                     region))
+
+    cluster_dict['module'][cloudtrail_module] = module_info
 
     return True

terraform/modules/tf_stream_alert_cloudtrail/main.tf

@@ -1,8 +1,10 @@
-// StreamAlert CloudTrail
+// StreamAlert CloudTrail, also sending to CloudWatch Logs group
 resource "aws_cloudtrail" "streamalert" {
-  count                         = "${var.existing_trail ? 0 : 1}"
+  count                         = "${var.send_to_cloudwatch && !var.existing_trail ? 1 : 0}"
   name                          = "${var.prefix}.${var.cluster}.streamalert.cloudtrail"
   s3_bucket_name                = "${aws_s3_bucket.cloudtrail_bucket.id}"
+  cloud_watch_logs_role_arn     = "${aws_iam_role.cloudtrail_to_cloudwatch_role.arn}"
+  cloud_watch_logs_group_arn    = "${aws_cloudwatch_log_group.cloudtrail_logging.arn}"
   enable_log_file_validation    = true
   enable_logging                = "${var.enable_logging}"
   include_global_service_events = true
@@ -22,6 +24,111 @@ resource "aws_cloudtrail" "streamalert" {
   }
 }
 
+// StreamAlert CloudTrail, not sending to CloudWatch
+resource "aws_cloudtrail" "streamalert_no_cloudwatch" {
+  count                         = "${!var.send_to_cloudwatch && !var.existing_trail ? 1 : 0}"
+  name                          = "${var.prefix}.${var.cluster}.streamalert.cloudtrail"
+  s3_bucket_name                = "${aws_s3_bucket.cloudtrail_bucket.id}"
+  enable_log_file_validation    = true
+  enable_logging                = "${var.enable_logging}"
+  include_global_service_events = true
+  is_multi_region_trail         = "${var.is_global_trail}"
+
+  event_selector {
+    read_write_type           = "All"
+    include_management_events = true
+
+    data_resource {
+      type = "AWS::S3::Object"
+
+      values = [
+        "arn:aws:s3",
+      ]
+    }
+  }
+}
+
+// CloudWatch Log group to send all CloudTrail logs to
+resource "aws_cloudwatch_log_group" "cloudtrail_logging" {
+  count             = "${var.send_to_cloudwatch ? 1 : 0}"
+  name              = "CloudTrail/DefaultLogGroup"
+  retention_in_days = 1
+}
+
+// IAM Role: Allow CloudTrail logs to send logs to CloudWatch Logs
+resource "aws_iam_role" "cloudtrail_to_cloudwatch_role" {
+  count = "${var.send_to_cloudwatch ? 1 : 0}"
+  name  = "cloudtrail_to_cloudwatch_role"
+
+  assume_role_policy = "${data.aws_iam_policy_document.cloudtrail_to_cloudwatch_assume_role_policy.json}"
+}
+
+// IAM Policy Document: Allow CloudTrail to AssumeRole
+data "aws_iam_policy_document" "cloudtrail_to_cloudwatch_assume_role_policy" {
+  count = "${var.send_to_cloudwatch ? 1 : 0}"
+
+  statement {
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["cloudtrail.amazonaws.com"]
+    }
+  }
+}
+
+// IAM Role Policy: Allow CloudTrail logs to create log streams and put logs to CloudWatch Logs
+resource "aws_iam_role_policy" "cloudtrail_to_cloudwatch_create_logs" {
+  count  = "${var.send_to_cloudwatch ? 1 : 0}"
+  name   = "CloudTrailToCloudWatchCreateLogs"
+  role   = "${aws_iam_role.cloudtrail_to_cloudwatch_role.id}"
+  policy = "${data.aws_iam_policy_document.cloudtrail_to_cloudwatch_create_logs.json}"
+}
+
+// IAM Policy Document: Allow CloudTrail logs to create log streams and put logs to CloudWatch Logs
+data "aws_iam_policy_document" "cloudtrail_to_cloudwatch_create_logs" {
+  count = "${var.send_to_cloudwatch ? 1 : 0}"
+
+  statement {
+    sid    = "AWSCloudTrailCreateLogStream"
+    effect = "Allow"
+
+    actions = [
+      "logs:CreateLogStream",
+    ]
+
+    resources = ["${aws_cloudwatch_log_group.cloudtrail_logging.arn}"]
+  }
+
+  statement {
+    sid    = "AWSCloudTrailPutLogEvents"
+    effect = "Allow"
+
+    actions = [
+      "logs:PutLogEvents",
+    ]
+
+    resources = ["${aws_cloudwatch_log_group.cloudtrail_logging.arn}"]
+  }
+}
+
+locals {
+  apply_filter_string = "{ $$.awsRegion != \"${var.region}\" }"
+}
+
+// CloudWatch Log Subscription Filter
+//   If we are collecting CloudTrail logs in the 'home region' another way, this allows
+//   for suppression of logs that originated in this region.
+resource "aws_cloudwatch_log_subscription_filter" "cloudtrail_via_cloudwatch" {
+  count           = "${var.send_to_cloudwatch ? 1 : 0}"
+  name            = "cloudtrail_delivery"
+  log_group_name  = "${aws_cloudwatch_log_group.cloudtrail_logging.name}"
+  filter_pattern  = "${var.exclude_home_region_events ? local.apply_filter_string : ""}"
+  role_arn        = "${var.subscription_role_arn}"
+  destination_arn = "${var.kinesis_arn}"
+  distribution    = "Random"
+}
+
 // S3 bucket for CloudTrail output
 resource "aws_s3_bucket" "cloudtrail_bucket" {
   count         = "${var.existing_trail ? 0 : 1}"

terraform/modules/tf_stream_alert_cloudtrail/variables.tf

@@ -39,3 +39,17 @@ variable "prefix" {
 variable "s3_logging_bucket" {
   type = "string"
 }
+
+variable "subscription_role_arn" {
+  default = ""
+}
+
+variable "send_to_cloudwatch" {
+  default = false
+}
+
+variable "exclude_home_region_events" {
+  default = false
+}
+
+variable "region" {}

tests/unit/stream_alert_cli/terraform/test_generate.py

@@ -327,6 +327,9 @@ def test_generate_cloudtrail_basic(self):
             's3_logging_bucket': 'unit-testing.streamalert.s3-logging',
             'existing_trail': False,
             'is_global_trail': True,
+            'region': 'us-west-1',
+            'exclude_home_region_events': False,
+            'send_to_cloudwatch': False,
             'event_pattern': '{"account": ["12345678910"]}'
         })
 
@@ -362,6 +365,9 @@ def test_generate_cloudtrail_all_options(self):
             'prefix': 'unit-testing',
             'enable_logging': True,
             'enable_kinesis': True,
+            'region': 'us-west-1',
+            'exclude_home_region_events': False,
+            'send_to_cloudwatch': False,
             'source': 'modules/tf_stream_alert_cloudtrail',
             's3_logging_bucket': 'unit-testing.streamalert.s3-logging',
             'event_pattern': '{"source": ["aws.ec2"], "account": "12345678910",'