↪️ Merge pull request Yelp#293 from Yelp/various_small_improvements

Various small improvements

.pre-commit-config.yaml

@@ -4,20 +4,23 @@ repos:
     hooks:
     -   id: check-builtin-literals
         args: ['--no-allow-dict-kwargs']
+        exclude: bumpity.py$
     -   id: check-docstring-first
     -   id: debug-statements
+        exclude: bumpity.py$
     -   id: double-quote-string-fixer
     -   id: end-of-file-fixer
     -   id: name-tests-test
     -   id: flake8
         args: ['--max-line-length', '100']
-        exclude: ^test_data/
+        exclude: ^test_data/|bumpity.py$
     -   id: trailing-whitespace
 -   repo: https://github.com/asottile/reorder_python_imports
     rev: v1.6.1
     hooks:
     -   id: reorder-python-imports
         language_version: python3
+        exclude: bumpity.py$
 -   repo: https://github.com/asottile/add-trailing-comma
     rev: v1.4.1
     hooks:

detect_secrets/core/bidirectional_iterator.py

@@ -1,4 +1,4 @@
-class BidirectionalIterator(object):
+class BidirectionalIterator:
     def __init__(self, collection):
         self.collection = collection
         self.index = -1  # Starts on -1, as index is increased _before_ getting result

detect_secrets/core/code_snippet.py

@@ -43,7 +43,7 @@ def get_code_snippet(self, file_lines, line_number, lines_of_context=5):
         )
 
 
-class CodeSnippet(object):
+class CodeSnippet:
 
     def __init__(self, snippet, start_line, target_index):
         """

detect_secrets/core/potential_secret.py

@@ -1,7 +1,7 @@
 import hashlib
 
 
-class PotentialSecret(object):
+class PotentialSecret:
     """This custom data type represents a string found, matching the
     plugin rules defined in SecretsCollection, that has the potential
     to be a secret that we actually care about.

detect_secrets/core/secrets_collection.py

@@ -13,7 +13,7 @@
 from detect_secrets.util import build_automaton
 
 
-class SecretsCollection(object):
+class SecretsCollection:
 
     def __init__(
         self,
@@ -156,6 +156,8 @@ def scan_diff(
         at incremental differences, rather than re-scanning the codebase every time.
         This function supports this, and adds information to self.data.
 
+        Note that this is only called by detect-secrets-server.
+
         :type diff: str
         :param diff: diff string.
                      e.g. The output of `git diff <fileA> <fileB>`
@@ -338,6 +340,7 @@ def _extract_secrets_from_patch(self, f, plugin, filename):
         """Extract secrets from a given patch file object.
 
         Note that we only want to capture incoming secrets (so added lines).
+        Note that this is only called by detect-secrets-server.
 
         :type f: unidiff.patch.PatchedFile
         :type plugin: detect_secrets.plugins.base.BasePlugin

detect_secrets/core/usage.py

@@ -42,7 +42,7 @@ def add_no_verify_flag(parser):
     )
 
 
-class ParserBuilder(object):
+class ParserBuilder:
 
     def __init__(self):
         self.parser = argparse.ArgumentParser()
@@ -133,7 +133,7 @@ def _add_no_verify_flag(self):
         return self
 
 
-class ScanOptions(object):
+class ScanOptions:
 
     def __init__(self, subparser):
         self.parser = subparser.add_parser(
@@ -209,7 +209,7 @@ def _add_adhoc_scanning_argument(self):
         )
 
 
-class AuditOptions(object):
+class AuditOptions:
 
     def __init__(self, subparser):
         self.parser = subparser.add_parser(
@@ -320,7 +320,7 @@ def get_disabled_help_text(plugin):
         return 'Disables {}'.format(line)
 
 
-class PluginOptions(object):
+class PluginOptions:
 
     all_plugins = [
         PluginDescriptor.from_plugin_class(plugin, name)

detect_secrets/plugins/base.py

@@ -24,7 +24,7 @@ def __get__(self, cls, owner):
         return classmethod(self.fget).__get__(None, owner)()
 
 
-class BasePlugin(object):
+class BasePlugin:
     """
     This is an abstract class to define Plugins API.
 
@@ -70,17 +70,15 @@ def __init__(
         :param false_positive_heuristics: List of fp-heuristic functions
         applicable to this plugin
         """
-        self.exclude_lines_regex = None
-        if exclude_lines_regex:
-            self.exclude_lines_regex = re.compile(exclude_lines_regex)
+        self.exclude_lines_regex = (
+            re.compile(exclude_lines_regex)
+            if exclude_lines_regex
+            else None
+        )
 
         self.should_verify = should_verify
 
-        self.false_positive_heuristics = (
-            false_positive_heuristics
-            if false_positive_heuristics
-            else []
-        )
+        self.false_positive_heuristics = false_positive_heuristics or []
 
     @classproperty
     def disable_flag_text(cls):
@@ -101,6 +99,19 @@ def disable_flag_text(cls):
     def default_options(cls):
         return {}
 
+    def _is_excluded_line(self, line):
+        return (
+            any(
+                allowlist_regex.search(line)
+                for allowlist_regex in ALLOWLIST_REGEXES
+            )
+            or
+            (
+                self.exclude_lines_regex and
+                self.exclude_lines_regex.search(line)
+            )
+        )
+
     def analyze(self, file, filename):
         """
         :param file:     The File object itself.
@@ -114,6 +125,13 @@ def analyze(self, file, filename):
         file_lines = tuple(file.readlines())
         for line_num, line in enumerate(file_lines, start=1):
             results = self.analyze_line(line, line_num, filename)
+            if (
+                not results
+                or
+                self._is_excluded_line(line)
+            ):
+                continue
+
             if not self.should_verify:
                 potential_secrets.update(results)
                 continue
@@ -146,18 +164,6 @@ def analyze_line(self, string, line_num, filename):
 
         NOTE: line_num and filename are used for PotentialSecret creation only.
         """
-        if (
-            any(
-                allowlist_regex.search(string) for allowlist_regex in ALLOWLIST_REGEXES
-            )
-
-            or (
-                self.exclude_lines_regex and
-                self.exclude_lines_regex.search(string)
-            )
-        ):
-            return {}
-
         return self.analyze_string_content(
             string,
             line_num,

detect_secrets/plugins/common/ini_file_parser.py

@@ -17,7 +17,7 @@ def append(self, lineno, line):
 configparser.ParsingError = EfficientParsingError
 
 
-class IniFileParser(object):
+class IniFileParser:
 
     _comment_regex = re.compile(r'\s*[;#]')
 

detect_secrets/plugins/common/initialize.py

@@ -165,7 +165,12 @@ def from_plugin_classname(
     try:
         klass = import_plugins()[plugin_classname]
     except KeyError:
-        log.warning('No such plugin to initialize.')
+        log.error('Error: No such `{}` plugin to initialize.'.format(plugin_classname))
+        log.error('Chances are you should run `pre-commit autoupdate`.')
+        log.error(
+            'This error occurs when using a baseline that was made by '
+            'a newer detect-secrets version than the one running.',
+        )
         raise TypeError
 
     try:

detect_secrets/plugins/common/yaml_file_parser.py

@@ -3,7 +3,7 @@
 from .constants import ALLOWLIST_REGEX
 
 
-class YamlFileParser(object):
+class YamlFileParser:
     """
     Yaml config files are interesting, because they don't necessarily conform
     to our basic regex for detecting HighEntropyStrings as strings don't

detect_secrets/pre_commit_hook.py

@@ -32,7 +32,7 @@ def main(argv=None):
         # If baseline is provided, we first want to make sure
         # it's valid, before doing any further computation.
         baseline_collection = get_baseline(args.baseline[0])
-    except (IOError, ValueError):
+    except (IOError, TypeError, ValueError):
         # Error logs handled within logic.
         return 1
 

scripts/benchmark.py

@@ -1,6 +1,4 @@
 #!/usr/bin/python3
-from __future__ import print_function
-
 import argparse
 import json
 import os

scripts/bumpity.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+"""Bumps the detect-secrets version,
+in both `detect_secrets/__init__.py` and `README.md`.
+
+Then commits.
+"""
+import argparse
+import pathlib
+import subprocess
+import sys
+
+
+PROJECT_ROOT = pathlib.Path(__file__).absolute().parent.parent
+INIT_FILE_PATH = PROJECT_ROOT.joinpath('detect_secrets/__init__.py')
+README_FILE_PATH = PROJECT_ROOT.joinpath('README.md')
+
+
+def _argparse_bump_type(value):
+    VALID_BUMP_TYPES = ('major', 'minor', 'patch')
+
+    if value in VALID_BUMP_TYPES:
+        return value
+
+    raise argparse.ArgumentTypeError(
+        f"Argument {value} must be one 'major', 'minor', 'patch'.",
+    )
+
+
+def parse_args(argv):
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        prog='bumpity',
+    )
+    parser.add_argument(
+        '--bump',
+        help='the bump type, specified as one of {major, minor, patch}',
+        metavar='{major,minor,patch}',
+        type=_argparse_bump_type,
+    )
+
+    return parser.parse_args(argv)
+
+
+def get_current_version():
+    with open(INIT_FILE_PATH) as init_file:
+        first_line = init_file.read().splitlines()[0]
+    # e.g. VERSION = '0.13.0'
+    _, semver = first_line.replace(' ', '').split('=')
+    return map(
+        int,
+        # e.g. '0.13.0'
+        semver.strip("'").split('.'),
+    )
+
+
+def update_init_file(new_version):
+    with open(INIT_FILE_PATH, 'w') as init_file:
+        init_file.write(f"VERSION = '{new_version}'\n")
+
+
+def update_readme(old_version, new_version):
+    with open(README_FILE_PATH, 'r') as readme:
+        original_text = readme.read()
+    with open(README_FILE_PATH, 'w') as readme:
+        readme.write(
+            original_text.replace(old_version, new_version),
+        )
+
+
+def stage_and_commit(new_version):
+    # Stage files
+    subprocess.check_output(
+        (
+            'git',
+            'add',
+            INIT_FILE_PATH,
+            README_FILE_PATH,
+        ),
+    )
+
+    # Check they are the only ones staged
+    staged_files = subprocess.check_output(
+        (
+            'git',
+            'diff',
+            '--staged',
+            '--name-only',
+        ),
+    ).splitlines()
+    if len(staged_files) != 2:
+        raise RuntimeWarning('More files staged than __init__.py and README.md')
+
+    # Make the commit
+    subprocess.check_output(
+        (
+            'git',
+            'commit',
+            '--message',
+            f':fist: Bumping version to {new_version}',
+            INIT_FILE_PATH,
+            README_FILE_PATH,
+        ),
+    )
+
+
+def main(argv=sys.argv[1:]):
+    if not argv:
+        argv.append('--help')
+    args = parse_args(argv)
+
+    major, minor, patch = get_current_version()
+    old_version = f'{major}.{minor}.{patch}'
+
+    if args.bump == 'major':
+        major += 1
+        minor = 0
+        patch = 0
+    elif args.bump == 'minor':
+        minor += 1
+        patch = 0
+    else:
+        patch += 1
+
+    new_version = f'{major}.{minor}.{patch}'
+    update_init_file(new_version)
+    update_readme(old_version, new_version)
+    stage_and_commit(new_version)
+    print("Don't forget to update CHANGELOG.md too!")
+
+
+if __name__ == '__main__':
+    sys.exit(main())

scripts/run_performance_tests.py

@@ -1,6 +1,3 @@
-#!/usr/bin/env python3
-from __future__ import print_function
-
 import argparse
 import json
 import os

testing/factories.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 from detect_secrets.core.potential_secret import PotentialSecret
 from detect_secrets.core.secrets_collection import SecretsCollection