Make terraform state and s3 logging buckets configurable (airbnb#770)

stoggi · Jun 20, 2018 · 245e9ba · 245e9ba
1 parent 68d063b
commit 245e9ba
Show file tree

Hide file tree

Showing 6 changed files with 62 additions and 34 deletions.
diff --git a/conf/global.json b/conf/global.json
@@ -35,9 +35,13 @@
       "write_capacity": 5
     }
   },
+  "s3_access_logging": {
+    "create_bucket": true,
+    "logging_bucket": "PREFIX_GOES_HERE.streamalert.s3-logging"
+  },
   "terraform": {
+    "create_bucket": true,
     "tfstate_bucket": "PREFIX_GOES_HERE.streamalert.terraform.state",
-    "tfstate_s3_key": "stream_alert_state/terraform.tfstate",
-    "tfvars": "terraform.tfvars"
+    "tfstate_s3_key": "stream_alert_state/terraform.tfstate"
   }
 }
diff --git a/stream_alert_cli/config.py b/stream_alert_cli/config.py
@@ -92,10 +92,19 @@ def set_prefix(self, prefix):
             LOGGER_CLI.error('Prefix cannot contain underscores')
             return
 
-        tf_state_bucket = '{}.streamalert.terraform.state'.format(prefix)
         self.config['global']['account']['prefix'] = prefix
         self.config['global']['account']['kms_key_alias'] = '{}_streamalert_secrets'.format(prefix)
-        self.config['global']['terraform']['tfstate_bucket'] = tf_state_bucket
+
+        # Set logging bucket name only if we will be creating it
+        if self.config['global']['s3_access_logging'].get('create_bucket', True):
+            self.config['global']['s3_access_logging']['logging_bucket'] = (
+                '{}.streamalert.s3-logging'.format(prefix))
+
+        # Set Terraform state bucket name only if we will be creating it
+        if self.config['global']['terraform'].get('create_bucket', True):
+            self.config['global']['terraform']['tfstate_bucket'] = (
+                '{}.streamalert.terraform.state'.format(prefix))
+
         self.config['lambda']['athena_partition_refresh_config']['buckets'].clear()
         self.config['lambda']['athena_partition_refresh_config']['buckets'] \
             ['{}.streamalerts'.format(prefix)] = 'alerts'

diff --git a/stream_alert_cli/terraform/generate.py b/stream_alert_cli/terraform/generate.py
@@ -111,44 +111,49 @@ def generate_main(config, init=False):
             'path': 'terraform.tfstate'}
     else:
         main_dict['terraform']['backend']['s3'] = {
-            'bucket': '{}.streamalert.terraform.state'.format(
-                config['global']['account']['prefix']),
-            'key': 'stream_alert_state/terraform.tfstate',
+            'bucket': config['global']['terraform']['tfstate_bucket'],
+            'key': config['global']['terraform']['tfstate_s3_key'],
             'region': config['global']['account']['region'],
             'encrypt': True,
             'acl': 'private',
             'kms_key_id': 'alias/{}'.format(config['global']['account']['kms_key_alias'])}
 
-    logging_bucket = '{}.streamalert.s3-logging'.format(
-        config['global']['account']['prefix'])
-    logging_bucket_lifecycle = {
-        'prefix': '/',
-        'enabled': True,
-        'transition': {
-            'days': 30,
-            'storage_class': 'GLACIER'}}
+    logging_bucket = config['global']['s3_access_logging']['logging_bucket']
 
     # Configure initial S3 buckets
     main_dict['resource']['aws_s3_bucket'] = {
         'stream_alert_secrets': generate_s3_bucket(
             bucket='{}.streamalert.secrets'.format(config['global']['account']['prefix']),
             logging=logging_bucket
         ),
-        'terraform_remote_state': generate_s3_bucket(
-            bucket=config['global']['terraform']['tfstate_bucket'],
+        'streamalerts': generate_s3_bucket(
+            bucket='{}.streamalerts'.format(config['global']['account']['prefix']),
             logging=logging_bucket
-        ),
-        'logging_bucket': generate_s3_bucket(
+        )
+    }
+
+    # Create bucket for S3 access logs (if applicable)
+    if config['global']['s3_access_logging'].get('create_bucket', True):
+        main_dict['resource']['aws_s3_bucket']['logging_bucket'] = generate_s3_bucket(
             bucket=logging_bucket,
             logging=logging_bucket,
             acl='log-delivery-write',
-            lifecycle_rule=logging_bucket_lifecycle
-        ),
-        'streamalerts': generate_s3_bucket(
-            bucket='{}.streamalerts'.format(config['global']['account']['prefix']),
+            lifecycle_rule={
+                'prefix': '/',
+                'enabled': True,
+                'transition': {
+                    'days': 365,
+                    'storage_class': 'GLACIER'
+                }
+            }
+        )
+
+    # Create bucket for Terraform state (if applicable)
+    if config['global']['terraform'].get('create_bucket', True):
+        main_dict['resource']['aws_s3_bucket']['terraform_remote_state'] = generate_s3_bucket(
+            bucket=config['global']['terraform']['tfstate_bucket'],
             logging=logging_bucket
         )
-    }
 
     # Setup Firehose Delivery Streams
     generate_firehose(config, main_dict, logging_bucket)

diff --git a/tests/unit/conf/global.json b/tests/unit/conf/global.json
@@ -32,8 +32,14 @@
       "write_capacity": 5
     }
   },
+  "s3_access_logging": {
+    "create_bucket": true,
+    "logging_bucket": "unit-testing.streamalert.s3-logging"
+  },
   "terraform": {
-    "tfstate_bucket": "unit-testing.terraform.tfstate"
+    "create_bucket": true,
+    "tfstate_bucket": "unit-testing.streamalert.terraform.state",
+    "tfstate_s3_key": "stream_alert_state/terraform.tfstate"
   },
   "threat_intel": {
     "dynamodb_table": "test_table_name",

diff --git a/tests/unit/helpers/base.py b/tests/unit/helpers/base.py
@@ -90,11 +90,6 @@ def basic_streamalert_config():
                 'prefix': 'unit-testing',
                 'region': 'us-west-2'
             },
-            'terraform': {
-                'tfstate_bucket': 'unit-testing.streamalert.terraform.state',
-                'tfstate_s3_key': 'stream_alert_state/terraform.tfstate',
-                'tfvars': 'terraform.tfvars'
-            },
             'infrastructure': {
                 'monitoring': {
                     'create_sns_topic': True,
@@ -112,7 +107,16 @@ def basic_streamalert_config():
                         }
                     }
                 }
-            }
+            },
+            's3_access_logging': {
+                'create_bucket': True,
+                'logging_bucket': 'unit-testing.streamalert.s3-logging'
+            },
+            'terraform': {
+                'create_bucket': True,
+                'tfstate_bucket': 'unit-testing.streamalert.terraform.state',
+                'tfstate_s3_key': 'stream_alert_state/terraform.tfstate'
+            },
         },
         'lambda': {
             'alert_merger_config': {

diff --git a/tests/unit/stream_alert_cli/terraform/test_generate.py b/tests/unit/stream_alert_cli/terraform/test_generate.py
@@ -126,15 +126,15 @@ def test_generate_main(self):
                         }
                     },
                     'terraform_remote_state': {
-                        'bucket': 'unit-testing.terraform.tfstate',
+                        'bucket': 'unit-testing.streamalert.terraform.state',
                         'acl': 'private',
                         'force_destroy': True,
                         'versioning': {
                             'enabled': True
                         },
                         'logging': {
                             'target_bucket': 'unit-testing.streamalert.s3-logging',
-                            'target_prefix': 'unit-testing.terraform.tfstate/'
+                            'target_prefix': 'unit-testing.streamalert.terraform.state/'
                         }
                     },
                     'logging_bucket': {
@@ -152,7 +152,7 @@ def test_generate_main(self):
                             'prefix': '/',
                             'enabled': True,
                             'transition': {
-                                'days': 30,
+                                'days': 365,
                                 'storage_class': 'GLACIER'
                             }
                         }