diff --git a/conf/global.json b/conf/global.json index 473bde4eb..45ae1a21c 100644 --- a/conf/global.json +++ b/conf/global.json @@ -35,9 +35,13 @@ "write_capacity": 5 } }, + "s3_access_logging": { + "create_bucket": true, + "logging_bucket": "PREFIX_GOES_HERE.streamalert.s3-logging" + }, "terraform": { + "create_bucket": true, "tfstate_bucket": "PREFIX_GOES_HERE.streamalert.terraform.state", - "tfstate_s3_key": "stream_alert_state/terraform.tfstate", - "tfvars": "terraform.tfvars" + "tfstate_s3_key": "stream_alert_state/terraform.tfstate" } } \ No newline at end of file diff --git a/stream_alert_cli/config.py b/stream_alert_cli/config.py index ee4c60b5d..828900e4c 100644 --- a/stream_alert_cli/config.py +++ b/stream_alert_cli/config.py @@ -92,10 +92,19 @@ def set_prefix(self, prefix): LOGGER_CLI.error('Prefix cannot contain underscores') return - tf_state_bucket = '{}.streamalert.terraform.state'.format(prefix) self.config['global']['account']['prefix'] = prefix self.config['global']['account']['kms_key_alias'] = '{}_streamalert_secrets'.format(prefix) - self.config['global']['terraform']['tfstate_bucket'] = tf_state_bucket + + # Set logging bucket name only if we will be creating it + if self.config['global']['s3_access_logging'].get('create_bucket', True): + self.config['global']['s3_access_logging']['logging_bucket'] = ( + '{}.streamalert.s3-logging'.format(prefix)) + + # Set Terraform state bucket name only if we will be creating it + if self.config['global']['terraform'].get('create_bucket', True): + self.config['global']['terraform']['tfstate_bucket'] = ( + '{}.streamalert.terraform.state'.format(prefix)) + self.config['lambda']['athena_partition_refresh_config']['buckets'].clear() self.config['lambda']['athena_partition_refresh_config']['buckets'] \ ['{}.streamalerts'.format(prefix)] = 'alerts' diff --git a/stream_alert_cli/terraform/generate.py b/stream_alert_cli/terraform/generate.py index 4fb5a7a87..20aa60b0e 100644 --- a/stream_alert_cli/terraform/generate.py +++ b/stream_alert_cli/terraform/generate.py @@ -111,22 +111,14 @@ def generate_main(config, init=False): 'path': 'terraform.tfstate'} else: main_dict['terraform']['backend']['s3'] = { - 'bucket': '{}.streamalert.terraform.state'.format( - config['global']['account']['prefix']), - 'key': 'stream_alert_state/terraform.tfstate', + 'bucket': config['global']['terraform']['tfstate_bucket'], + 'key': config['global']['terraform']['tfstate_s3_key'], 'region': config['global']['account']['region'], 'encrypt': True, 'acl': 'private', 'kms_key_id': 'alias/{}'.format(config['global']['account']['kms_key_alias'])} - logging_bucket = '{}.streamalert.s3-logging'.format( - config['global']['account']['prefix']) - logging_bucket_lifecycle = { - 'prefix': '/', - 'enabled': True, - 'transition': { - 'days': 30, - 'storage_class': 'GLACIER'}} + logging_bucket = config['global']['s3_access_logging']['logging_bucket'] # Configure initial S3 buckets main_dict['resource']['aws_s3_bucket'] = { @@ -134,21 +126,34 @@ def generate_main(config, init=False): bucket='{}.streamalert.secrets'.format(config['global']['account']['prefix']), logging=logging_bucket ), - 'terraform_remote_state': generate_s3_bucket( - bucket=config['global']['terraform']['tfstate_bucket'], + 'streamalerts': generate_s3_bucket( + bucket='{}.streamalerts'.format(config['global']['account']['prefix']), logging=logging_bucket - ), - 'logging_bucket': generate_s3_bucket( + ) + } + + # Create bucket for S3 access logs (if applicable) + if config['global']['s3_access_logging'].get('create_bucket', True): + main_dict['resource']['aws_s3_bucket']['logging_bucket'] = generate_s3_bucket( bucket=logging_bucket, logging=logging_bucket, acl='log-delivery-write', - lifecycle_rule=logging_bucket_lifecycle - ), - 'streamalerts': generate_s3_bucket( - bucket='{}.streamalerts'.format(config['global']['account']['prefix']), + lifecycle_rule={ + 'prefix': '/', + 'enabled': True, + 'transition': { + 'days': 365, + 'storage_class': 'GLACIER' + } + } + ) + + # Create bucket for Terraform state (if applicable) + if config['global']['terraform'].get('create_bucket', True): + main_dict['resource']['aws_s3_bucket']['terraform_remote_state'] = generate_s3_bucket( + bucket=config['global']['terraform']['tfstate_bucket'], logging=logging_bucket ) - } # Setup Firehose Delivery Streams generate_firehose(config, main_dict, logging_bucket) diff --git a/tests/unit/conf/global.json b/tests/unit/conf/global.json index ab6a9f946..6d2e97381 100644 --- a/tests/unit/conf/global.json +++ b/tests/unit/conf/global.json @@ -32,8 +32,14 @@ "write_capacity": 5 } }, + "s3_access_logging": { + "create_bucket": true, + "logging_bucket": "unit-testing.streamalert.s3-logging" + }, "terraform": { - "tfstate_bucket": "unit-testing.terraform.tfstate" + "create_bucket": true, + "tfstate_bucket": "unit-testing.streamalert.terraform.state", + "tfstate_s3_key": "stream_alert_state/terraform.tfstate" }, "threat_intel": { "dynamodb_table": "test_table_name", diff --git a/tests/unit/helpers/base.py b/tests/unit/helpers/base.py index 8265c020e..a5a85cae6 100644 --- a/tests/unit/helpers/base.py +++ b/tests/unit/helpers/base.py @@ -90,11 +90,6 @@ def basic_streamalert_config(): 'prefix': 'unit-testing', 'region': 'us-west-2' }, - 'terraform': { - 'tfstate_bucket': 'unit-testing.streamalert.terraform.state', - 'tfstate_s3_key': 'stream_alert_state/terraform.tfstate', - 'tfvars': 'terraform.tfvars' - }, 'infrastructure': { 'monitoring': { 'create_sns_topic': True, @@ -112,7 +107,16 @@ def basic_streamalert_config(): } } } - } + }, + 's3_access_logging': { + 'create_bucket': True, + 'logging_bucket': 'unit-testing.streamalert.s3-logging' + }, + 'terraform': { + 'create_bucket': True, + 'tfstate_bucket': 'unit-testing.streamalert.terraform.state', + 'tfstate_s3_key': 'stream_alert_state/terraform.tfstate' + }, }, 'lambda': { 'alert_merger_config': { diff --git a/tests/unit/stream_alert_cli/terraform/test_generate.py b/tests/unit/stream_alert_cli/terraform/test_generate.py index dbcaff521..47aad17b0 100644 --- a/tests/unit/stream_alert_cli/terraform/test_generate.py +++ b/tests/unit/stream_alert_cli/terraform/test_generate.py @@ -126,7 +126,7 @@ def test_generate_main(self): } }, 'terraform_remote_state': { - 'bucket': 'unit-testing.terraform.tfstate', + 'bucket': 'unit-testing.streamalert.terraform.state', 'acl': 'private', 'force_destroy': True, 'versioning': { @@ -134,7 +134,7 @@ def test_generate_main(self): }, 'logging': { 'target_bucket': 'unit-testing.streamalert.s3-logging', - 'target_prefix': 'unit-testing.terraform.tfstate/' + 'target_prefix': 'unit-testing.streamalert.terraform.state/' } }, 'logging_bucket': { @@ -152,7 +152,7 @@ def test_generate_main(self): 'prefix': '/', 'enabled': True, 'transition': { - 'days': 30, + 'days': 365, 'storage_class': 'GLACIER' } }