Skip to content

Commit

Permalink
official release of v3.4.0 (airbnb#1292)
Browse files Browse the repository at this point in the history
* bumping version to 3.4.0

* updating to terraform v0.13.0 (airbnb#1290)

* updating terraform version to 0.13.0 and aws provider to 3.3.0

* misc updates to terraform code

* fixes

* updating for vpc flow logs

* patching out writing of vars in tests

* cloudtrail to cwl fix

* fix
  • Loading branch information
ryandeivert authored Aug 26, 2020
1 parent bfde778 commit b7971a0
Showing 14 changed files with 78 additions and 65 deletions.
2 changes: 1 addition & 1 deletion Vagrantfile
Original file line number Diff line number Diff line change
@@ -59,7 +59,7 @@ def configure_streamalert(machine)
privileged: false
end

TERRAFORM_VERSION = ENV.fetch('SA_TERRAFORM_VERSION', '0.12.9')
TERRAFORM_VERSION = ENV.fetch('SA_TERRAFORM_VERSION', '0.13.0')
def configure_terraform(machine)
# Install terraform with the specified version.
machine.vm.provision :shell,
8 changes: 4 additions & 4 deletions docs/source/getting-started.rst
Original file line number Diff line number Diff line change
@@ -10,17 +10,17 @@ Install Dependencies
********************

1. Install Python 3.7 and `pip <https://pip.pypa.io/en/stable/installing/>`_
2. Install `Terraform <https://www.terraform.io/intro/getting-started/install.html>`_ >= v0.12.9:
2. Install `Terraform <https://www.terraform.io/intro/getting-started/install.html>`_ >= v0.13.0:

.. code-block:: bash
brew install terraform # MacOS Homebrew
terraform --version # Must be >= v0.12.9
terraform --version # Must be >= v0.13.0
.. note::

Terraform versions lower than 0.12 are not supported. We recommend installing Terraform
version 0.12.9 or greater.
Terraform versions lower than 0.13 are not supported. We recommend installing Terraform
version 0.13.0 or greater.


3. If you are using Linux, you may need to install the Python development libraries:
2 changes: 1 addition & 1 deletion streamalert/__init__.py
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
"""StreamAlert version."""
__version__ = '3.3.0'
__version__ = '3.4.0'
10 changes: 10 additions & 0 deletions streamalert_cli/_infrastructure/_include.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
terraform {
required_version = "~> 0.13.0"

required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 3.3.0"
}
}
}
Original file line number Diff line number Diff line change
@@ -3,6 +3,14 @@
// define its own provider block.
// See here for additional reading:
// https://www.terraform.io/docs/configuration/modules.html#providers-within-modules

// Default provider
// TODO: this should be updated to use required_providers:
// https://www.terraform.io/docs/configuration/provider-requirements.html#requiring-providers
provider "aws" {
region = var.region
}

provider "aws" {
alias = "ap-northeast-1"
region = "ap-northeast-1"
3 changes: 3 additions & 0 deletions streamalert_cli/_infrastructure/_variables.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
variable "region" {
type = string
}
Original file line number Diff line number Diff line change
@@ -66,9 +66,9 @@ data "aws_iam_policy_document" "firehose_s3" {
// IAM Policy: Interact with the Glue Catalog
resource "aws_iam_role_policy" "streamalert_firehose_glue" {
name = "FirehoseReadGlueCatalog"
role = "${aws_iam_role.streamalert_kinesis_firehose.id}"
role = aws_iam_role.streamalert_kinesis_firehose.id

policy = "${data.aws_iam_policy_document.firehose_glue_catalog.json}"
policy = data.aws_iam_policy_document.firehose_glue_catalog.json
}

// IAM Policy Document: Interact with the Glue Catalog
@@ -77,7 +77,7 @@ data "aws_iam_policy_document" "firehose_glue_catalog" {
effect = "Allow"

actions = [
"glue:GetTableVersions"
"glue:GetTable*"
]

resources = ["*"]
Original file line number Diff line number Diff line change
@@ -52,14 +52,14 @@ data "aws_iam_policy_document" "cloudtrail_to_cloudwatch_create_logs" {
sid = "AWSCloudTrailCreateLogStream"
effect = "Allow"
actions = ["logs:CreateLogStream"]
resources = [aws_cloudwatch_log_group.cloudtrail_logging.arn]
resources = ["${aws_cloudwatch_log_group.cloudtrail_logging.arn}:log-stream:*"]
}

statement {
sid = "AWSCloudTrailPutLogEvents"
effect = "Allow"
actions = ["logs:PutLogEvents"]
resources = [aws_cloudwatch_log_group.cloudtrail_logging.arn]
resources = ["${aws_cloudwatch_log_group.cloudtrail_logging.arn}:log-stream:*"]
}
}

Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@ output "cloudtrail_to_cloudwatch_logs_role" {
value = aws_iam_role.cloudtrail_to_cloudwatch_role.arn
}

// CloudTrail requires the log stream wildcard here
output "cloudwatch_logs_group_arn" {
value = aws_cloudwatch_log_group.cloudtrail_logging.arn
value = "${aws_cloudwatch_log_group.cloudtrail_logging.arn}:*"
}
40 changes: 21 additions & 19 deletions streamalert_cli/_infrastructure/modules/tf_flow_logs/main.tf
Original file line number Diff line number Diff line change
@@ -1,25 +1,28 @@
resource "aws_flow_log" "vpc_flow_log" {
count = length(var.vpcs)
vpc_id = element(var.vpcs, count.index)
log_group_name = aws_cloudwatch_log_group.flow_log_group.name
iam_role_arn = aws_iam_role.flow_log_role.arn
traffic_type = "ALL"
count = length(var.vpcs)
vpc_id = element(var.vpcs, count.index)
log_destination = aws_cloudwatch_log_group.flow_log_group.arn
log_destination_type = "cloud-watch-logs"
iam_role_arn = aws_iam_role.flow_log_role.arn
traffic_type = "ALL"
}

resource "aws_flow_log" "subnet_flow_log" {
count = length(var.subnets)
subnet_id = element(var.subnets, count.index)
log_group_name = aws_cloudwatch_log_group.flow_log_group.name
iam_role_arn = aws_iam_role.flow_log_role.arn
traffic_type = "ALL"
count = length(var.subnets)
subnet_id = element(var.subnets, count.index)
log_destination = aws_cloudwatch_log_group.flow_log_group.arn
log_destination_type = "cloud-watch-logs"
iam_role_arn = aws_iam_role.flow_log_role.arn
traffic_type = "ALL"
}

resource "aws_flow_log" "eni_flow_log" {
count = length(var.enis)
eni_id = element(var.enis, count.index)
log_group_name = aws_cloudwatch_log_group.flow_log_group.name
iam_role_arn = aws_iam_role.flow_log_role.arn
traffic_type = "ALL"
count = length(var.enis)
eni_id = element(var.enis, count.index)
log_destination = aws_cloudwatch_log_group.flow_log_group.arn
log_destination_type = "cloud-watch-logs"
iam_role_arn = aws_iam_role.flow_log_role.arn
traffic_type = "ALL"
}

resource "aws_cloudwatch_log_group" "flow_log_group" {
@@ -34,8 +37,7 @@ resource "aws_cloudwatch_log_group" "flow_log_group" {

resource "aws_cloudwatch_log_subscription_filter" "flow_logs" {
name = "${aws_cloudwatch_log_group.flow_log_group.name}_to_kinesis"
log_group_name = "${aws_cloudwatch_log_group.flow_log_group.name}"
filter_pattern = "${var.flow_log_filter}"
destination_arn = "${var.cloudwatch_logs_destination_arn}"
log_group_name = aws_cloudwatch_log_group.flow_log_group.name
filter_pattern = var.flow_log_filter
destination_arn = var.cloudwatch_logs_destination_arn
}

Original file line number Diff line number Diff line change
@@ -54,9 +54,9 @@ data "aws_iam_policy_document" "firehose_s3" {
// IAM Policy: Interact with the Glue Catalog
resource "aws_iam_role_policy" "stream_alert_firehose_glue" {
name = "streamalert_firehose_read_glue_catalog"
role = "${aws_iam_role.firehose.id}"
role = aws_iam_role.firehose.id

policy = "${data.aws_iam_policy_document.firehose_glue_catalog.json}"
policy = data.aws_iam_policy_document.firehose_glue_catalog.json
}

// IAM Policy Document: Interact with the Glue Catalog
Original file line number Diff line number Diff line change
@@ -88,9 +88,9 @@ data "aws_iam_policy_document" "firehose_cloudwatch" {
// IAM Policy: Interact with the Glue Catalog
resource "aws_iam_role_policy" "streamalert_firehose_glue" {
name = "streamalert_firehose_read_glue_catalog"
role = "${aws_iam_role.streamalert_kinesis_firehose.id}"
role = aws_iam_role.streamalert_kinesis_firehose.id

policy = "${data.aws_iam_policy_document.firehose_glue_catalog.json}"
policy = data.aws_iam_policy_document.firehose_glue_catalog.json
}

// IAM Policy Document: Interact with the Glue Catalog
@@ -99,7 +99,7 @@ data "aws_iam_policy_document" "firehose_glue_catalog" {
effect = "Allow"

actions = [
"glue:GetTableVersions"
"glue:GetTable*"
]

resources = ["*"]
28 changes: 13 additions & 15 deletions streamalert_cli/terraform/generate.py
Original file line number Diff line number Diff line change
@@ -63,24 +63,20 @@
from streamalert_cli.utils import CLICommand

RESTRICTED_CLUSTER_NAMES = ('main', 'athena')
TERRAFORM_VERSION = '~> 0.12.9'
TERRAFORM_PROVIDER_VERSION = '~> 2.48.0'

LOGGER = get_logger(__name__)


def _terraform_defaults(region):
return infinitedict({
'terraform': {
'required_version': TERRAFORM_VERSION,
},
'provider': {
'aws': {
'region': region,
'version': TERRAFORM_PROVIDER_VERSION,
},
},
})
def write_vars(config, **kwargs):
"""Write root variables to a terraform.tfvars.json file
Keyword Args:
region (string): AWS region where infrastructure will be built
"""
_create_terraform_module_file(
kwargs,
os.path.join(config.build_directory, 'terraform.tfvars.json')
)


def generate_s3_bucket(bucket, logging, **kwargs):
@@ -164,7 +160,9 @@ def generate_main(config, init=False):
Returns:
dict: main.tf.json Terraform dict
"""
main_dict = _terraform_defaults(config['global']['account']['region'])
write_vars(config, region=config['global']['account']['region'])

main_dict = infinitedict()

logging_bucket, create_logging_bucket = s3_access_logging_bucket(config)

19 changes: 5 additions & 14 deletions tests/unit/streamalert_cli/terraform/test_generate.py
Original file line number Diff line number Diff line change
@@ -13,7 +13,7 @@
See the License for the specific language governing permissions and
limitations under the License.
"""
from mock import ANY, patch
from mock import ANY, Mock, patch

from nose.tools import (
assert_equal,
@@ -35,6 +35,7 @@
)


@patch('streamalert_cli.terraform.generate.write_vars', Mock())
class TestTerraformGenerate:
"""Test class for the Terraform Cluster Generating"""
# pylint: disable=no-self-use,attribute-defined-outside-init
@@ -44,8 +45,7 @@ def setup(self):
self.cluster_dict = common.infinitedict()
self.config = CLIConfig(config_path='tests/unit/conf')

@staticmethod
def test_generate_s3_bucket():
def test_generate_s3_bucket(self):
"""CLI - Terraform Generate S3 Bucket """
bucket = generate.generate_s3_bucket(
bucket='unit.test.bucket',
@@ -67,8 +67,7 @@ def test_generate_s3_bucket():
assert_equal(bucket['bucket'], 'unit.test.bucket')
assert_equal(set(bucket.keys()), required_keys)

@staticmethod
def test_generate_s3_bucket_lifecycle():
def test_generate_s3_bucket_lifecycle(self):
"""CLI - Terraform Generate S3 Bucket with Lifecycle"""
bucket = generate.generate_s3_bucket(
bucket='unit.test.bucket',
@@ -91,14 +90,7 @@ def test_generate_main(self):
tf_main = generate.generate_main(config=self.config, init=False)

tf_main_expected = {
'provider': {
'aws': {
'version': '~> 2.48.0', # Changes to this should require unit test update
'region': 'us-west-1'
}
},
'terraform': {
'required_version': '~> 0.12.9', # Changes to this should require unit test update
'backend': {
's3': {
'bucket': 'unit-test-streamalert-terraform-state',
@@ -229,7 +221,6 @@ def test_generate_main(self):
}
}

assert_dict_equal(tf_main['provider'], tf_main_expected['provider'])
assert_dict_equal(tf_main['terraform'], tf_main_expected['terraform'])
assert_dict_equal(tf_main['resource'], tf_main_expected['resource'])

@@ -389,7 +380,7 @@ def test_generate_cloudtrail_with_s3_events(self):
"""CLI - Terraform Generate CloudTrail Module, With S3 Events"""
cluster_name = 'advanced'
self.config['clusters']['advanced']['modules']['cloudtrail'] = {
's3_settings':{
's3_settings': {
'bucket_name': 'unit-test-bucket',
'cross_account_ids': ['456789012345'],
'enable_events': True,

0 comments on commit b7971a0

Please sign in to comment.