Merge pull request Yelp#391 from pablosantiagolopez/feature/v1-multip…

…le-filters

Adding --exclude-secrets flag to explicitly ignore secret values

README.md

@@ -266,6 +266,8 @@ filter options:
                         If lines match this regex, it will be ignored.
   --exclude-files EXCLUDE_FILES
                         If filenames match this regex, it will be ignored.
+  --exclude-secrets EXCLUDE_SECRETS
+                        If secrets match this regex, it will be ignored.
   --word-list WORD_LIST_FILE
                         Text file with a list of words, if a secret contains a
                         word in the list we ignore it.
@@ -318,6 +320,8 @@ filter options:
                         If lines match this regex, it will be ignored.
   --exclude-files EXCLUDE_FILES
                         If filenames match this regex, it will be ignored.
+  --exclude-secrets EXCLUDE_SECRETS
+                        If secrets match this regex, it will be ignored.
   --word-list WORD_LIST_FILE
                         Text file with a list of words, if a secret contains a
                         word in the list we ignore it.
@@ -430,6 +434,12 @@ specific pattern. You can specify a regex rule as such:
 $ detect-secrets scan --exclude-lines 'password = (blah|fake)'
 ```
 
+Or you can specify multiple regex rules as such:
+
+```bash
+$ detect-secrets scan --exclude-lines 'password = blah' --exclude-lines 'password = fake'
+```
+
 #### --exclude-files
 
 Sometimes, you want to be able to ignore certain files in your scan. You can specify a regex
@@ -439,6 +449,27 @@ pattern to do so, and if the filename meets this regex pattern, it will not be s
 $ detect-secrets scan --exclude-files '.*\.signature$'
 ```
 
+Or you can specify multiple regex patterns as such:
+
+```bash
+$ detect-secrets scan --exclude-files '.*\.signature$' --exclude-files '.*/i18n/.*'
+```
+
+#### --exclude-secrets
+
+Sometimes, you want to be able to ignore certain secret values in your scan. You can specify
+a regex rule as such:
+
+```bash
+$ detect-secrets scan --exclude-secrets '(fakesecret|\${.*})'
+```
+
+Or you can specify multiple regex rules as such:
+
+```bash
+$ detect-secrets scan --exclude-secrets 'fakesecret' --exclude-secrets '\${.*})'
+```
+
 #### --word-list
 
 If you know there are certain fake password values that you want to ignore, you can also use

detect_secrets/core/upgrades/v1_0.py

@@ -58,13 +58,17 @@ def _migrate_filters(baseline: Dict[str, Any]) -> None:
         if baseline['exclude'].get('files'):
             baseline['filters_used'].append({
                 'path': 'detect_secrets.filters.regex.should_exclude_file',
-                'pattern': baseline['exclude']['files'],
+                'pattern': [
+                    baseline['exclude']['files'],
+                ],
             })
 
         if baseline['exclude'].get('lines'):
             baseline['filters_used'].append({
                 'path': 'detect_secrets.filters.regex.should_exclude_line',
-                'pattern': baseline['exclude']['lines'],
+                'pattern': [
+                    baseline['exclude']['lines'],
+                ],
             })
 
         baseline.pop('exclude')

detect_secrets/core/usage/filters.py

@@ -38,13 +38,21 @@ def add_filter_options(parent: argparse.ArgumentParser) -> None:
     parser.add_argument(
         '--exclude-lines',
         type=str,
+        action='append',
         help='If lines match this regex, it will be ignored.',
     )
     parser.add_argument(
         '--exclude-files',
         type=str,
+        action='append',
         help='If filenames match this regex, it will be ignored.',
     )
+    parser.add_argument(
+        '--exclude-secrets',
+        type=str,
+        action='append',
+        help='If secrets match this regex, it will be ignored.',
+    )
 
     if filters.wordlist.is_feature_enabled():
         parser.add_argument(
@@ -126,6 +134,11 @@ def parse_args(args: argparse.Namespace) -> None:
             'pattern': args.exclude_files,
         }
 
+    if args.exclude_secrets:
+        get_settings().filters['detect_secrets.filters.regex.should_exclude_secret'] = {
+            'pattern': args.exclude_secrets,
+        }
+
     if (
         filters.wordlist.is_feature_enabled()
         and args.word_list_file

detect_secrets/filters/regex.py

@@ -1,28 +1,49 @@
 import re
 from functools import lru_cache
+from typing import List
 from typing import Pattern
 
 from ..settings import get_settings
 from .util import get_caller_path
 
 
 def should_exclude_line(line: str) -> bool:
-    regex = _get_line_exclusion_regex()
-    return bool(regex.search(line))
+    regexes = _get_line_exclusion_regex()
+    for regex in regexes:
+        if regex.search(line):
+            return True
+    return False
 
 
 @lru_cache(maxsize=1)
-def _get_line_exclusion_regex() -> Pattern:
+def _get_line_exclusion_regex() -> List[Pattern]:
     path = get_caller_path(offset=1)
-    return re.compile(get_settings().filters[path]['pattern'])
+    return [re.compile(regex) for regex in get_settings().filters[path]['pattern']]
 
 
 def should_exclude_file(filename: str) -> bool:
-    regex = _get_file_exclusion_regex()
-    return bool(regex.search(filename))
+    regexes = _get_file_exclusion_regex()
+    for regex in regexes:
+        if regex.search(filename):
+            return True
+    return False
 
 
 @lru_cache(maxsize=1)
-def _get_file_exclusion_regex() -> Pattern:
+def _get_file_exclusion_regex() -> List[Pattern]:
     path = get_caller_path(offset=1)
-    return re.compile(get_settings().filters[path]['pattern'])
+    return [re.compile(regex) for regex in get_settings().filters[path]['pattern']]
+
+
+def should_exclude_secret(secret: str) -> bool:
+    regexes = _get_secret_exclusion_regex()
+    for regex in regexes:
+        if regex.search(secret):
+            return True
+    return False
+
+
+@lru_cache(maxsize=1)
+def _get_secret_exclusion_regex() -> List[Pattern]:
+    path = get_caller_path(offset=1)
+    return [re.compile(regex) for regex in get_settings().filters[path]['pattern']]

docs/filters.md

@@ -52,6 +52,7 @@ the `detect_secrets.filters` namespace.
 | `heuristic.is_non_text_file`                     | Ignores non-text files (e.g. archives, images).                                     |
 | `regex.should_exclude_line`                      | Powers the [`--exclude-lines` functionality](../README.md#--exclude-lines).         |
 | `regex.should_exclude_file`                      | Powers the [`--exclude-files` functionality](../README.md#--exclude-files).         |
+| `regex.should_exclude_secret`                    | Powers the [`--exclude-secrets` functionality](../README.md#--exclude-secrets).     |
 | `wordlist.should_exclude_secret`                 | Powers the [`--word-list` functionality](../README.md#--word-list).                 |
 
 ## Configuring Filters

tests/core/secrets_collection_test.py

@@ -55,7 +55,9 @@ def test_line_based_success():
             # This gets rid of the aws keys with `EXAMPLE` in them.
             {
                 'path': 'detect_secrets.filters.regex.should_exclude_line',
-                'pattern': 'EXAMPLE',
+                'pattern': [
+                    'EXAMPLE',
+                ],
             },
         ])
 
@@ -135,7 +137,9 @@ def test_filename_filters_are_invoked_first():
         get_settings().configure_filters([
             {
                 'path': 'detect_secrets.filters.regex.should_exclude_file',
-                'pattern': 'test|baseline',
+                'pattern': [
+                    'test|baseline',
+                ],
             },
         ])
 
@@ -350,7 +354,9 @@ def test_subtraction(configure_plugins):
         'filters_used': [
             {
                 'path': 'detect_secrets.filters.regex.should_exclude_line',
-                'pattern': 'EXAMPLE',
+                'pattern': [
+                    'EXAMPLE',
+                ],
             },
         ],
     }):

tests/filters/regex_filter_test.py

@@ -14,9 +14,14 @@ def parser():
 
 
 def test_should_exclude_line(parser):
-    parser.parse_args(['--exclude-lines', 'canarytoken'])
+    parser.parse_args([
+        '--exclude-lines', 'canarytoken',
+        '--exclude-lines', '^not-real-secret = .*$',
+    ])
     assert filters.regex.should_exclude_line('password = "canarytoken"') is True
     assert filters.regex.should_exclude_line('password = "hunter2"') is False
+    assert filters.regex.should_exclude_line('not-real-secret = value') is True
+    assert filters.regex.should_exclude_line('maybe-not-real-secret = value') is False
 
     assert [
         item
@@ -25,15 +30,23 @@ def test_should_exclude_line(parser):
     ] == [
         {
             'path': 'detect_secrets.filters.regex.should_exclude_line',
-            'pattern': 'canarytoken',
+            'pattern': [
+                'canarytoken',
+                '^not-real-secret = .*$',
+            ],
         },
     ]
 
 
 def test_should_exclude_file(parser):
-    parser.parse_args(['--exclude-files', '^tests/.*'])
+    parser.parse_args([
+        '--exclude-files', '^tests/.*',
+        '--exclude-files', '.*/i18/.*',
+    ])
     assert filters.regex.should_exclude_file('tests/blah.py') is True
     assert filters.regex.should_exclude_file('detect_secrets/tests/blah.py') is False
+    assert filters.regex.should_exclude_file('app/messages/i18/en.properties') is True
+    assert filters.regex.should_exclude_file('app/i18secrets/secrets.yaml') is False
 
     assert [
         item
@@ -42,6 +55,34 @@ def test_should_exclude_file(parser):
     ] == [
         {
             'path': 'detect_secrets.filters.regex.should_exclude_file',
-            'pattern': '^tests/.*',
+            'pattern': [
+                '^tests/.*',
+                '.*/i18/.*',
+            ],
+        },
+    ]
+
+
+def test_should_exclude_secret(parser):
+    parser.parse_args([
+        '--exclude-secrets', '^[Pp]assword[0-9]{0,3}$',
+        '--exclude-secrets', 'my-first-password',
+    ])
+    assert filters.regex.should_exclude_secret('Password123') is True
+    assert filters.regex.should_exclude_secret('MyRealPassword') is False
+    assert filters.regex.should_exclude_secret('1-my-first-password-for-database') is True
+    assert filters.regex.should_exclude_secret('my-password') is False
+
+    assert [
+        item
+        for item in get_settings().json()['filters_used']
+        if item['path'] == 'detect_secrets.filters.regex.should_exclude_secret'
+    ] == [
+        {
+            'path': 'detect_secrets.filters.regex.should_exclude_secret',
+            'pattern': [
+                '^[Pp]assword[0-9]{0,3}$',
+                'my-first-password',
+            ],
         },
     ]