[tf][cli] add cross-account cloudwatch logs support (airbnb#744)

* [tf] adding terraform module for performing cross account ingestion of cloudwatch logs

* [cli] adding tf generate code for cloudwatch module

* [test] adding tf cloudwatch generate tests

* addressing pr feedback

stream_alert_cli/terraform/cloudwatch.py

@@ -0,0 +1,76 @@
+"""
+Copyright 2017-present, Airbnb Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from stream_alert_cli.logger import LOGGER_CLI
+
+
+def generate_cloudwatch(cluster_name, cluster_dict, config):
+    """Add the CloudWatch destinations, mapping to the configured kinesis stream
+
+    Args:
+        cluster_name (str): The name of the currently generating cluster
+        cluster_dict (defaultdict): The dict containing all Terraform config for
+                                    a given cluster.
+        config (dict): The loaded config from the 'conf/' directory
+
+    Returns:
+        bool: Result of applying the cloudwatch module
+    """
+    cloudwatch_module = config['clusters'][cluster_name]['modules']['cloudwatch']
+
+    if not cloudwatch_module.get('enabled', True):
+        LOGGER_CLI.info('The \'cloudwatch\' module is not enabled, nothing to do.')
+        return True
+
+    # Ensure that the kinesis module is enabled for this cluster since the
+    # cloudwatch module will utilize the created stream for sending data
+    if not config['clusters'][cluster_name]['modules'].get('kinesis'):
+        LOGGER_CLI.error('The \'kinesis\' module must be enabled to enable the '
+                         '\'cloudwatch\' module.')
+        return False
+
+    account_id = config['global']['account']['aws_account_id']
+    cross_account_ids = cloudwatch_module.get('cross_account_ids', []) + [account_id]
+    excluded_regions = set(cloudwatch_module.get('excluded_regions', set()))
+
+    # Exclude any desired regions from the entire list of regions
+    regions = {
+        'ap-northeast-1',
+        'ap-northeast-2',
+        'ap-south-1',
+        'ap-southeast-1',
+        'ap-southeast-2',
+        'ca-central-1',
+        'eu-central-1',
+        'eu-west-1',
+        'eu-west-2',
+        'eu-west-3',
+        'sa-east-1',
+        'us-east-1',
+        'us-east-2',
+        'us-west-1',
+        'us-west-2',
+    }.difference(excluded_regions)
+
+    for region in regions:
+        cluster_dict['module']['cloudwatch_{}_{}'.format(cluster_name, region)] = {
+            'source': 'modules/tf_stream_alert_cloudwatch',
+            'region': region,
+            'cross_account_ids': cross_account_ids,
+            'cluster': cluster_name,
+            'kinesis_stream_arn': '${{module.kinesis_{}.arn}}'.format(cluster_name)
+        }
+
+    return True

stream_alert_cli/terraform/generate.py

@@ -31,6 +31,7 @@
 from stream_alert_cli.terraform.app_integrations import generate_app_integrations
 from stream_alert_cli.terraform.athena import generate_athena
 from stream_alert_cli.terraform.cloudtrail import generate_cloudtrail
+from stream_alert_cli.terraform.cloudwatch import generate_cloudwatch
 from stream_alert_cli.terraform.firehose import generate_firehose
 from stream_alert_cli.terraform.flow_logs import generate_flow_logs
 from stream_alert_cli.terraform.kinesis_events import generate_kinesis_events
@@ -282,18 +283,19 @@ def generate_cluster(config, cluster_name):
         if not generate_kinesis_events(cluster_name, cluster_dict, config):
             return
 
-    cloudtrail_info = modules.get('cloudtrail')
-    if cloudtrail_info:
+    if modules.get('cloudtrail'):
         if not generate_cloudtrail(cluster_name, cluster_dict, config):
             return
 
-    flow_log_info = modules.get('flow_logs')
-    if flow_log_info:
+    if modules.get('cloudwatch'):
+        if not generate_cloudwatch(cluster_name, cluster_dict, config):
+            return
+
+    if modules.get('flow_logs'):
         if not generate_flow_logs(cluster_name, cluster_dict, config):
             return
 
-    s3_events_info = modules.get('s3_events')
-    if s3_events_info:
+    if modules.get('s3_events'):
         if not generate_s3_events(cluster_name, cluster_dict, config):
             return
 

terraform/modules/tf_stream_alert_cloudwatch/README.md

@@ -0,0 +1,59 @@
+# StreamAlert CloudWatch Logs Cross-Account Terraform Module
+* This Terraform module enables cross-account collection of CloudWatch Logs, via a CloudWatch Logs destination in each region.
+* This module leverages the [concepts found here.]('https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/CrossAccountSubscriptions.html')
+
+## Components
+Creates the following in _each region_:
+* IAM role that will grant CloudWatch Logs the permission to put data into your Kinesis stream.
+* Permissions policy to define which actions CloudWatch Logs can perform.
+  * `kinesis:PutRecord` for the Kinesis stream
+  * `iam:PassRole` for the previously created IAM role
+* A CloudWatch Log destination that points to the cluster's default Kinesis stream.
+* Policy that defines who has write access to the destination
+  * `logs:PutSubscriptionFilter` for the cross-account principals (account IDs)
+
+
+## Example
+```
+module "cloudwatch_prod_us-west-1" {
+  source                 = "modules/tf_stream_alert_cloudwatch"
+  cluster                = "prod"
+  kinesis_stream_arn     = "${module.kinesis_advanced.arn}"
+  cross_account_ids      = ["123456789012", "12345678910"]
+  region                 = "us-west-1"
+}
+```
+
+## Inputs
+<table>
+  <tr>
+    <th>Property</th>
+    <th>Description</th>
+    <th>Default</th>
+    <th>Required</th>
+  </tr>
+  <tr>
+    <td>cluster</td>
+    <td>Name of the cluster</td>
+    <td>None</td>
+    <td>True</td>
+  </tr>
+  <tr>
+    <td>kinesis_stream_arn</td>
+    <td>ARN of the Kinesis Stream which receives the CloudWatch Logs. Output from the tf_stream_alert_kinesis_streams module</td>
+    <td>None</td>
+    <td>True</td>
+  </tr>
+  <tr>
+    <td>cross_account_ids</td>
+    <td>List of AWS Account IDs for which to enable cross-account log collection</td>
+    <td>[]</td>
+    <td>False</td>
+  </tr>
+  <tr>
+    <td>region</td>
+    <td>The AWS region of your VPC(s), Subnet(s), or ENI(s)</td>
+    <td>None</td>
+    <td>True</td>
+  </tr>
+</table>

terraform/modules/tf_stream_alert_cloudwatch/iam.tf

@@ -0,0 +1,87 @@
+# IAM Role: Allows CloudWatch Logs to put data into
+# this cluster's default Kinesis stream
+resource "aws_iam_role" "cloudwatch_subscription_role" {
+  name = "stream_alert_${var.cluster}_cloudwatch_subscription_role_${var.region}"
+
+  assume_role_policy = "${data.aws_iam_policy_document.cloudwatch_logs_assume_role_policy.json}"
+}
+
+// IAM Policy Document: AssumeRole for CloudWatch Logs
+data "aws_iam_policy_document" "cloudwatch_logs_assume_role_policy" {
+  statement {
+    effect  = "Allow"
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["logs.${var.region}.amazonaws.com"]
+    }
+  }
+}
+
+// IAM Policy: Write to Kinesis
+resource "aws_iam_role_policy" "cloudwatch_kinesis_wo" {
+  name = "WriteCWLogsToKinesis"
+  role = "${aws_iam_role.cloudwatch_subscription_role.id}"
+
+  policy = "${data.aws_iam_policy_document.cloudwatch_put_kinesis_events.json}"
+}
+
+// IAM Policy Document: Write to Kinesis
+data "aws_iam_policy_document" "cloudwatch_put_kinesis_events" {
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "kinesis:PutRecord",
+    ]
+
+    resources = [
+      "${var.kinesis_stream_arn}",
+    ]
+  }
+
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "iam:PassRole",
+    ]
+
+    resources = [
+      "${aws_iam_role.cloudwatch_subscription_role.arn}",
+    ]
+  }
+}
+
+# IAM Policy: Access policy to allow writing CloudWatch logs cross-account
+resource "aws_cloudwatch_log_destination_policy" "cloudwatch_kinesis" {
+  count            = "${length(var.cross_account_ids) > 0 ? 1 : 0}"
+  destination_name = "${aws_cloudwatch_log_destination.cloudwatch_kinesis.name}"
+  access_policy    = "${data.aws_iam_policy_document.cross_account_destination_policy.json}"
+}
+
+// IAM Policy Document: Allow Cross Account CloudWatch logs subscription
+data "aws_iam_policy_document" "cross_account_destination_policy" {
+  count = "${length(var.cross_account_ids) > 0 ? 1 : 0}"
+
+  statement {
+    effect = "Allow"
+
+    principals = {
+      type = "AWS"
+
+      identifiers = [
+        "${var.cross_account_ids}",
+      ]
+    }
+
+    actions = [
+      "logs:PutSubscriptionFilter",
+    ]
+
+    resources = [
+      "${var.kinesis_stream_arn}",
+    ]
+  }
+}

terraform/modules/tf_stream_alert_cloudwatch/main.tf

@@ -0,0 +1,11 @@
+provider "aws" {
+  region = "${var.region}"
+}
+
+# CloudWatch Logs Destination
+# Sends logs to the default Kinesis stream for this cluster
+resource "aws_cloudwatch_log_destination" "cloudwatch_kinesis" {
+  name       = "stream_alert_${var.cluster}_cloudwatch_to_kinesis"
+  role_arn   = "${aws_iam_role.cloudwatch_subscription_role.arn}"
+  target_arn = "${var.kinesis_stream_arn}"
+}

terraform/modules/tf_stream_alert_cloudwatch/variables.tf

@@ -0,0 +1,10 @@
+variable "cluster" {}
+
+variable "kinesis_stream_arn" {}
+
+variable "region" {}
+
+variable "cross_account_ids" {
+  type    = "list"
+  default = []
+}

tests/unit/conf/clusters/advanced.json

@@ -5,6 +5,15 @@
       "enable_kinesis": true,
       "enable_logging": true
     },
+    "cloudwatch": {
+      "cross_account_ids": [
+        "123456789012"
+      ],
+      "enabled": true,
+      "excluded_regions": [
+        "us-west-1"
+      ]
+    },
     "cloudwatch_monitoring": {
       "enabled": true,
       "settings": {

tests/unit/stream_alert_cli/terraform/test_generate.py

@@ -19,6 +19,7 @@
 from stream_alert_cli.terraform import (
     common,
     cloudtrail,
+    cloudwatch,
     flow_logs,
     generate,
     streamalert
@@ -30,11 +31,7 @@
 
 class TestTerraformGenerate(object):
     """Test class for the Terraform Cluster Generating"""
-    # pylint: disable=no-self-use
-
-    def __init__(self):
-        self.cluster_dict = None
-        self.config = None
+    # pylint: disable=no-self-use,attribute-defined-outside-init
 
     def setup(self):
         """Setup before each method"""
@@ -388,6 +385,30 @@ def test_generate_cloudtrail_invalid_event_pattern(self, mock_logging):
         assert_false(result)
         assert_true(mock_logging.error.called)
 
+    def test_generate_cloudwatch(self):
+        """CLI - Terraform Generate CloudWatch"""
+        cloudwatch.generate_cloudwatch(
+            'advanced',
+            self.cluster_dict,
+            self.config
+        )
+
+        # Count the modules for each region - there should be 14 since 1 is excluded
+        count = sum(1 for name in self.cluster_dict['module']
+                    if name.startswith('cloudwatch_advanced'))
+        assert_equal(count, 14)
+
+        expected_config = {
+            'cluster': 'advanced',
+            'source': 'modules/tf_stream_alert_cloudwatch',
+            'region': 'eu-west-1',
+            'kinesis_stream_arn': '${module.kinesis_advanced.arn}',
+            'cross_account_ids': ['123456789012', '12345678910']
+        }
+
+        eu_west_config = self.cluster_dict['module']['cloudwatch_advanced_eu-west-1']
+        assert_equal(expected_config, eu_west_config)
+
     def test_generate_cluster_test(self):
         """CLI - Terraform Generate Test Cluster"""
 
@@ -424,6 +445,20 @@ def test_generate_cluster_advanced(self):
 
         advanced_modules = {
             'stream_alert_advanced',
+            'cloudwatch_advanced_eu-west-1',
+            'cloudwatch_advanced_eu-west-2',
+            'cloudwatch_advanced_eu-west-3',
+            'cloudwatch_advanced_us-west-2',
+            'cloudwatch_advanced_sa-east-1',
+            'cloudwatch_advanced_eu-central-1',
+            'cloudwatch_advanced_ap-northeast-2',
+            'cloudwatch_advanced_ap-northeast-1',
+            'cloudwatch_advanced_ap-southeast-1',
+            'cloudwatch_advanced_ca-central-1',
+            'cloudwatch_advanced_ap-southeast-2',
+            'cloudwatch_advanced_us-east-1',
+            'cloudwatch_advanced_us-east-2',
+            'cloudwatch_advanced_ap-south-1',
             'cloudwatch_monitoring_advanced',
             'kinesis_advanced',
             'kinesis_events_advanced',