Add alert UUID and use it in SNS subject (airbnb#626)

rules/community/binaryalert/binaryalert_yara_match.py

@@ -1,4 +1,4 @@
-"""Alert on destructive AWS API calls."""
+"""Alert on BinaryAlert YARA matches"""
 from stream_alert.rule_processor.rules_engine import StreamRules
 
 rule = StreamRules.rule

stream_alert/alert_processor/helpers.py

@@ -31,6 +31,7 @@ def validate_alert(alert):
         return False
 
     alert_keys = {
+        'id',
         'record',
         'rule_name',
         'rule_description',
@@ -79,3 +80,20 @@ def validate_alert(alert):
             valid = False
 
     return valid
+
+
+def elide_string_middle(text, max_length):
+    """Replace the middle of the text with ellipses to shorten text to the desired length.
+
+    Args:
+        text (str): Text to shorten.
+        max_length (int): Maximum allowable length of the string.
+
+    Returns:
+        (str) The elided text, e.g. "Some really long tex ... the end."
+    """
+    if len(text) <= max_length:
+        return text
+
+    half_len = (max_length - 5) / 2  # Length of text on either side.
+    return '{} ... {}'.format(text[:half_len], text[-half_len:])

stream_alert/alert_processor/main.py

@@ -62,6 +62,7 @@ def run(alert, region, account_id, function_name, config):
             following structure:
 
             {
+                'id': uuid,
                 'record': record,
                 'rule_name': rule.rule_name,
                 'rule_description': rule.rule_function.__doc__,
@@ -84,7 +85,7 @@ def run(alert, region, account_id, function_name, config):
         LOGGER.error('Invalid alert format:\n%s', json.dumps(alert, indent=2))
         return
 
-    LOGGER.debug('Sending alert to outputs:\n%s', json.dumps(alert, indent=2))
+    LOGGER.info('Sending alert %s to outputs %s', alert['id'], alert['outputs'])
 
     # strip out unnecessary keys and sort
     alert = _sort_dict(alert)

stream_alert/alert_processor/outputs/aws.py

@@ -24,6 +24,7 @@
 import boto3
 
 from stream_alert.alert_processor import LOGGER
+from stream_alert.alert_processor.helpers import elide_string_middle
 from stream_alert.alert_processor.outputs.output_base import (
     OutputDispatcher,
     OutputProperty,
@@ -336,7 +337,13 @@ def dispatch(self, **kwargs):
         topic_arn = 'arn:aws:sns:{}:{}:{}'.format(self.region, self.account_id, topic_name)
         topic = boto3.resource('sns', region_name=self.region).Topic(topic_arn)
 
-        response = topic.publish(Message=json.dumps(kwargs['alert'], indent=2))
+        alert = kwargs['alert']
+        response = topic.publish(
+            Message=json.dumps(alert, indent=2, sort_keys=True),
+            # Subject must be < 100 characters long
+            Subject=elide_string_middle(
+                '{} triggered alert {}'.format(kwargs['rule_name'], alert['id']), 99)
+        )
         return self._log_status(response, kwargs['descriptor'])
 
 

stream_alert/rule_processor/rules_engine.py

@@ -15,6 +15,7 @@
 """
 from collections import namedtuple
 from copy import copy
+import uuid
 
 from stream_alert.rule_processor import LOGGER
 from stream_alert.rule_processor.threat_intel import StreamThreatIntel
@@ -438,10 +439,12 @@ def rule_analysis(record, rule, payload, alerts):
             if StreamRules.check_alerts_duplication(record, rule, alerts):
                 return
 
-            LOGGER.info('Rule [%s] triggered an alert on log type [%s] from entity \'%s\' '
-                        'in service \'%s\'', rule.rule_name, payload.log_source,
+            alert_id = str(uuid.uuid4())  # Random unique alert ID
+            LOGGER.info('Rule [%s] triggered alert [%s] on log type [%s] from entity \'%s\' '
+                        'in service \'%s\'', rule.rule_name, alert_id, payload.log_source,
                         payload.entity, payload.service())
             alert = {
+                'id': alert_id,
                 'record': record,
                 'rule_name': rule.rule_name,
                 'rule_description': rule.rule_function.__doc__ or DEFAULT_RULE_DESCRIPTION,

tests/unit/stream_alert_alert_processor/helpers.py

@@ -69,6 +69,7 @@ def get_alert(context=None):
         context(dict): context dictionary (None by default)
     """
     return {
+        'id': '79192344-4a6d-4850-8d06-9c3fef1060a4',
         'record': {
             'compressed_size': '9982',
             'timestamp': '1496947381.18',

tests/unit/stream_alert_alert_processor/test_helpers.py

@@ -13,9 +13,9 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-from nose.tools import assert_false, assert_true
+from nose.tools import assert_equal, assert_false, assert_true
 
-from stream_alert.alert_processor.helpers import validate_alert
+from stream_alert.alert_processor.helpers import elide_string_middle, validate_alert
 from tests.unit.stream_alert_alert_processor.helpers import get_alert
 
 
@@ -100,3 +100,18 @@ def test_metadata_non_string_type():
 
     # Test with invalid metadata non-string value
     assert_false(validate_alert(invalid_metadata_non_string))
+
+
+def test_elide_string_middle():
+    """Alert Processor String Truncation"""
+    alphabet = 'abcdefghijklmnopqrstuvwxyz'
+
+    # String shortened
+    assert_equal('ab ... yz', elide_string_middle(alphabet, 10))
+    assert_equal('abcde ... vwxyz', elide_string_middle(alphabet, 15))
+    assert_equal('abcdefg ... tuvwxyz', elide_string_middle(alphabet, 20))
+    assert_equal('abcdefghij ... qrstuvwxyz', elide_string_middle(alphabet, 25))
+
+    # String unchanged
+    assert_equal(alphabet, elide_string_middle(alphabet, 26))
+    assert_equal(alphabet, elide_string_middle(alphabet, 50))

tests/unit/stream_alert_rule_processor/test_rules_engine.py

@@ -99,6 +99,7 @@ def alert_format_test(rec):  # pylint: disable=unused-variable
         alerts, _ = self.rules_engine.process(payload)
 
         alert_keys = {
+            'id',
             'record',
             'rule_name',
             'rule_description',
@@ -110,6 +111,7 @@ def alert_format_test(rec):  # pylint: disable=unused-variable
             'context'
         }
         assert_items_equal(alerts[0].keys(), alert_keys)
+        assert_is_instance(alerts[0]['id'], str)
         assert_is_instance(alerts[0]['record'], dict)
         assert_is_instance(alerts[0]['outputs'], list)
         assert_is_instance(alerts[0]['context'], dict)