Merge branch 'master' of github.com:Yelp/detect-secrets into verifiab…

…le-secrets

.pre-commit-config.yaml

@@ -2,15 +2,17 @@ repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v2.1.0
     hooks:
-    -   id: trailing-whitespace
-    -   id: end-of-file-fixer
+    -   id: check-builtin-literals
+        args: ['--no-allow-dict-kwargs']
     -   id: check-docstring-first
     -   id: debug-statements
+    -   id: double-quote-string-fixer
+    -   id: end-of-file-fixer
     -   id: name-tests-test
-        exclude: tests/util
     -   id: flake8
         args: ['--max-line-length', '100']
         exclude: ^test_data/
+    -   id: trailing-whitespace
 -   repo: https://github.com/asottile/reorder_python_imports
     rev: v1.3.4
     hooks:

detect_secrets/core/audit.py

@@ -23,8 +23,8 @@
 class SecretNotFoundOnSpecifiedLineError(Exception):
     def __init__(self, line):
         super(SecretNotFoundOnSpecifiedLineError, self).__init__(
-            "ERROR: Secret not found on line {}!\n".format(line) +
-            "Try recreating your baseline to fix this issue.",
+            'ERROR: Secret not found on line {}!\n'.format(line) +
+            'Try recreating your baseline to fix this issue.',
         )
 
 

detect_secrets/core/baseline.py

@@ -5,9 +5,11 @@
 import re
 import subprocess
 
+from detect_secrets import util
 from detect_secrets.core.log import get_logger
 from detect_secrets.core.secrets_collection import SecretsCollection
 
+
 log = get_logger(format_string='%(message)s')
 
 
@@ -37,17 +39,19 @@ def initialize(
         exclude_lines=exclude_lines_regex,
     )
 
-    files_to_scan = list()
+    files_to_scan = []
     for element in path:
         if os.path.isdir(element):
             if should_scan_all_files:
                 files_to_scan.extend(_get_files_recursively(element))
             else:
-                files_to_scan.extend(_get_git_tracked_files(element))
+                files = _get_git_tracked_files(element)
+                if files:
+                    files_to_scan.extend(files)
         elif os.path.isfile(element):
             files_to_scan.append(element)
         else:
-            log.error("detect-secrets: " + element + ": No such file or directory")
+            log.error('detect-secrets: ' + element + ': No such file or directory')
 
     if not files_to_scan:
         return output
@@ -211,7 +215,7 @@ def merge_results(old_results, new_results):
         if filename not in new_results:
             continue
 
-        old_secrets_mapping = dict()
+        old_secrets_mapping = {}
         for old_secret in old_secrets:
             old_secrets_mapping[old_secret['hashed_secret']] = old_secret
 
@@ -268,13 +272,16 @@ def _get_git_tracked_files(rootdir='.'):
             git_files = subprocess.check_output(
                 [
                     'git',
+                    '-C', rootdir,
                     'ls-files',
-                    rootdir,
                 ],
                 stderr=fnull,
             )
 
-        return set(git_files.decode('utf-8').split())
+        return set([
+            util.get_relative_path(rootdir, filename)
+            for filename in git_files.decode('utf-8').split()
+        ])
     except subprocess.CalledProcessError:
         return None
 
@@ -284,8 +291,8 @@ def _get_files_recursively(rootdir):
     This function allows us to do so.
     """
     output = []
-    for root, dirs, files in os.walk(rootdir):
+    for root, _, files in os.walk(rootdir):
         for filename in files:
-            output.append(os.path.join(root, filename))
+            output.append(util.get_relative_path(root, filename))
 
     return output

detect_secrets/core/potential_secret.py

@@ -110,8 +110,8 @@ def __hash__(self):
 
     def __str__(self):  # pragma: no cover
         return (
-            "Secret Type: %s\n"
-            "Location:    %s:%d\n"
+            'Secret Type: %s\n'
+            'Location:    %s:%d\n'
         ) % (
             self.type,
             self.filename, self.lineno,

detect_secrets/core/secrets_collection.py

@@ -212,7 +212,7 @@ def scan_file(self, filename, filename_key=None):
 
             return True
         except IOError:
-            log.warning("Unable to open file: %s", filename)
+            log.warning('Unable to open file: %s', filename)
             return False
 
     def get_secret(self, filename, secret, type_=None):
@@ -266,7 +266,7 @@ def format_for_baseline_output(self):
         plugins_used = sorted(plugins_used, key=lambda x: x['name'])
 
         return {
-            'generated_at': strftime("%Y-%m-%dT%H:%M:%SZ", gmtime()),
+            'generated_at': strftime('%Y-%m-%dT%H:%M:%SZ', gmtime()),
             'exclude': {
                 'files': self.exclude_files,
                 'lines': self.exclude_lines,
@@ -305,14 +305,14 @@ def _extract_secrets_from_file(self, f, filename):
         :type filename: string
         """
         try:
-            log.info("Checking file: %s", filename)
+            log.info('Checking file: %s', filename)
 
             for results, plugin in self._results_accumulator(filename):
                 results.update(plugin.analyze(f, filename))
                 f.seek(0)
 
         except UnicodeDecodeError:
-            log.warning("%s failed to load.", filename)
+            log.warning('%s failed to load.', filename)
 
     def _extract_secrets_from_patch(self, f, plugin, filename):
         """Extract secrets from a given patch file object.

detect_secrets/main.py

@@ -84,7 +84,7 @@ def main(argv=None):
 
 def _get_plugin_from_baseline(old_baseline):
     plugins = []
-    if old_baseline and "plugins_used" in old_baseline:
+    if old_baseline and 'plugins_used' in old_baseline:
         secrets_collection = SecretsCollection.load_baseline_from_dict(old_baseline)
         plugins = secrets_collection.plugins
     return plugins

detect_secrets/plugins/artifactory.py

@@ -11,7 +11,7 @@ class ArtifactoryDetector(RegexBasedDetector):
 
     denylist = [
         # artifactory tokens begin with AKC
-        re.compile(r'(\s|=|:|"|^)AKC\w{10,}'),    # api token
+        re.compile(r'(?:\s|=|:|"|^)AKC\w{10,}'),    # api token
         # artifactory encrypted passwords begin with AP6
-        re.compile(r'(\s|=|:|"|^)AP6\w{10,}'),    # password
+        re.compile(r'(?:\s|=|:|"|^)AP6\w{10,}'),    # password
     ]

detect_secrets/plugins/base.py

@@ -3,10 +3,11 @@
 from abc import abstractmethod
 from abc import abstractproperty
 
+from .common.constants import ALLOWLIST_REGEXES
+from .common.filters import is_false_positive
 from detect_secrets.core.code_snippet import CodeSnippetHighlighter
 from detect_secrets.core.constants import VerifiedResult
 from detect_secrets.core.potential_secret import PotentialSecret
-from detect_secrets.plugins.common.constants import ALLOWLIST_REGEXES
 
 
 # NOTE: In this whitepaper (Section V-D), it suggests that there's an
@@ -233,4 +234,7 @@ def analyze_string_content(self, string, line_num, filename):
     def secret_generator(self, string, *args, **kwargs):
         for regex in self.denylist:
             for match in regex.findall(string):
+                if is_false_positive(match):
+                    continue
+
                 yield match

detect_secrets/plugins/common/filters.py

@@ -0,0 +1,45 @@
+"""
+Heuristic, false positive filters that are shared across all plugin types.
+This abstraction allows for development of later ML work, or further
+heuristical determinations (e.g. word filter, entropy comparator).
+"""
+import string
+
+
+def is_false_positive(secret):
+    for func in [
+        is_sequential_string,
+    ]:
+        if func(secret):
+            return True
+
+    return False
+
+
+def is_sequential_string(secret):
+    """
+    Returns true if string is sequential.
+    """
+    sequences = (
+        (
+            string.ascii_uppercase +
+            string.ascii_uppercase +
+            string.digits +
+            string.ascii_uppercase +
+            string.ascii_uppercase +
+            '+/'
+        ),
+
+        # Capturing any number sequences
+        '0123456789' * 2,
+
+        string.hexdigits.upper() + string.hexdigits.upper(),
+        string.ascii_uppercase + '=/',
+    )
+
+    uppercase = secret.upper()
+    for sequential_string in sequences:
+        if uppercase in sequential_string:
+            return True
+
+    return False

detect_secrets/plugins/common/initialize.py

@@ -85,7 +85,7 @@ def _remove_key(d, key):
         return r
 
     baseline_plugins_dict = {
-        vars(plugin)["name"]: _remove_key(vars(plugin), "name")
+        vars(plugin)['name']: _remove_key(vars(plugin), 'name')
         for plugin in baseline_plugins
     }
 
@@ -134,7 +134,7 @@ def _remove_key(d, key):
         except KeyError:
             log.warning(
                 '{} specified, but {} not configured! Ignoring...',
-                "".join(["--", param_name.replace("_", "-")]),
+                ''.join(['--', param_name.replace('_', '-')]),
                 plugin_name,
             )
 

detect_secrets/plugins/high_entropy_strings.py

@@ -14,23 +14,12 @@
 import yaml
 
 from .base import BasePlugin
+from .common.filters import is_false_positive
+from .common.ini_file_parser import IniFileParser
+from .common.yaml_file_parser import YamlFileParser
 from detect_secrets.core.potential_secret import PotentialSecret
-from detect_secrets.plugins.common.ini_file_parser import IniFileParser
-from detect_secrets.plugins.common.yaml_file_parser import YamlFileParser
-
-
-IGNORED_SEQUENTIAL_STRINGS = (
-    (
-        string.ascii_uppercase +
-        string.ascii_uppercase +
-        string.digits +
-        string.ascii_uppercase +
-        string.ascii_uppercase +
-        '+/'
-    ),
-    string.hexdigits.upper() + string.hexdigits.upper(),
-    string.ascii_uppercase + '=/',
-)
+
+
 YAML_EXTENSIONS = (
     '.yaml',
     '.yml',
@@ -97,22 +86,16 @@ def calculate_shannon_entropy(self, data):
 
         return entropy
 
-    def _is_sequential_string(self, string):
-        uppercased_string = string.upper()
-        for sequential_string in IGNORED_SEQUENTIAL_STRINGS:
-            if uppercased_string in sequential_string:
-                return True
-        return False
-
     def analyze_string_content(self, string, line_num, filename):
         """Searches string for custom pattern, and captures all high entropy strings that
         match self.regex, with a limit defined as self.entropy_limit.
         """
         output = {}
 
         for result in self.secret_generator(string):
-            if self._is_sequential_string(result):
+            if is_false_positive(result):
                 continue
+
             secret = PotentialSecret(self.secret_type, filename, result, line_num)
             output[secret] = secret
 

detect_secrets/plugins/stripe.py

@@ -14,5 +14,5 @@ class StripeDetector(RegexBasedDetector):
 
     denylist = (
         # stripe standard keys begin with sk_live and restricted with rk_live
-        re.compile(r'(r|s)k_live_[0-9a-zA-Z]{24}'),
+        re.compile(r'(?:r|s)k_live_[0-9a-zA-Z]{24}'),
     )

detect_secrets/util.py

@@ -8,3 +8,10 @@ def get_root_directory():       # pragma: no cover
             '../',
         ),
     )
+
+
+def get_relative_path(root, path):
+    """Returns relative path, after following symlinks."""
+    return os.path.realpath(
+        os.path.join(root, path),
+    )[len(os.getcwd() + '/'):]

setup.py

@@ -12,7 +12,7 @@
     long_description=(
         'Check out detect-secrets on `GitHub <https://github.com/Yelp/detect-secrets>`_!'
     ),
-    license="Copyright Yelp, Inc. 2018",
+    license='Copyright Yelp, Inc. 2018',
     author='Aaron Loo',
     author_email='[email protected]',
     url='https://github.com/Yelp/detect-secrets',
@@ -37,14 +37,14 @@
         ],
     },
     classifiers=[
-        "Programming Language :: Python :: 2",
-        "Programming Language :: Python :: 3",
-        "License :: OSI Approved :: Apache Software License",
-        "Intended Audience :: Developers",
-        "Topic :: Software Development",
-        "Topic :: Utilities",
-        "Environment :: Console",
-        "Operating System :: OS Independent",
-        "Development Status :: 5 - Production/Stable",
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 3',
+        'License :: OSI Approved :: Apache Software License',
+        'Intended Audience :: Developers',
+        'Topic :: Software Development',
+        'Topic :: Utilities',
+        'Environment :: Console',
+        'Operating System :: OS Independent',
+        'Development Status :: 5 - Production/Stable',
     ],
 )

test_data/config.ini

@@ -1,13 +1,13 @@
 [credentials]
-password = 12345678901234
+password = 123456789a1234
 
 [parent]
     [child]
     keyA = 678912345
     keyB = value1
 
 [aws]
-aws_secret_key = 2345678901
+aws_secret_key = 23456789a1
 
 [key with multiple values]
 keyA =

test_data/files/file_with_no_secrets.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-REGULAR_VALUE = "this is just a long string, like a user facing error message"
+REGULAR_VALUE = 'this is just a long string, like a user facing error message'
 
 
 def main():

test_data/files/tmp/file_with_no_secrets.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python
 # Will change this later.
-SUPER_SECRET_VALUE = "this is just a long string, like a user facing error message"
+SUPER_SECRET_VALUE = 'this is just a long string, like a user facing error message'
 
 
 def main():