Merge pull request Yelp#440 from Yelp/feature/support-scans-from-diff…

…erent-directories

feature: adding ability to scan different directories

README.md

@@ -44,6 +44,12 @@ Create a baseline of potential secrets currently found in your git repository.
 $ detect-secrets scan > .secrets.baseline
 ```
 
+or, to run it from a different directory:
+
+```bash
+$ detect-secrets -C /path/to/directory scan > /path/to/directory/.secrets.baseline
+```
+
 **Scanning non-git tracked files:**
 
 ```bash

detect_secrets/core/baseline.py

@@ -18,11 +18,15 @@
 from .secrets_collection import SecretsCollection
 
 
-def create(*paths: str, should_scan_all_files: bool = False) -> SecretsCollection:
+def create(*paths: str, should_scan_all_files: bool = False, root: str = '') -> SecretsCollection:
     """Scans all the files recursively in path to initialize a baseline."""
-    secrets = SecretsCollection()
+    secrets = SecretsCollection(root=root)
 
-    for filename in get_files_to_scan(*paths, should_scan_all_files=should_scan_all_files):
+    for filename in get_files_to_scan(
+        *paths,
+        should_scan_all_files=should_scan_all_files,
+        root=root,
+    ):
         secrets.scan_file(filename)
 
     return secrets

detect_secrets/core/scan.py

@@ -19,15 +19,16 @@
 from ..util import git
 from ..util.code_snippet import get_code_snippet
 from ..util.inject import call_function_with_arguments
-from ..util.path import get_relative_path_if_in_cwd
+from ..util.path import get_relative_path
 from .log import log
 from .plugins import Plugin
 from .potential_secret import PotentialSecret
 
 
 def get_files_to_scan(
     *paths: str,
-    should_scan_all_files: bool = False
+    should_scan_all_files: bool = False,
+    root: str = '',
 ) -> Generator[str, None, None]:
     """
     If we specify specific files, we should be able to scan them. This abides by the
@@ -49,7 +50,12 @@ def get_files_to_scan(
     the scan for all files.
 
     See test cases for more details.
+
+    :param root: if not specified, will assume current repository as root.
     """
+    if root:
+        root = os.path.realpath(root)
+
     # First, we determine the appropriate filtering mode to be used.
     # If this is True, then it will consider everything to be valid.
     # Otherwise, it will only list the files that are valid.
@@ -62,7 +68,7 @@ def get_files_to_scan(
 
         if not should_scan_all_files:
             try:
-                valid_paths = git.get_tracked_files(git.get_root_directory())
+                valid_paths = git.get_tracked_files(git.get_root_directory(root))
             except subprocess.CalledProcessError:
                 log.warning('Did not detect git repository. Try scanning all files instead.')
                 valid_paths = False
@@ -77,14 +83,17 @@ def get_files_to_scan(
 
     for path in paths:
         iterator = (
-            cast(List[Tuple], [(os.getcwd(), None, [path])])
+            cast(List[Tuple], [(root or os.getcwd(), None, [path])])
             if os.path.isfile(path)
             else os.walk(path)
         )
 
         for path_root, _, filenames in iterator:
             for filename in filenames:
-                relative_path = get_relative_path_if_in_cwd(os.path.join(path_root, filename))
+                relative_path = get_relative_path(
+                    root=root or os.getcwd(),
+                    path=os.path.join(path_root, filename),
+                )
                 if not relative_path:
                     # e.g. symbolic links may be pointing outside the root directory
                     continue

detect_secrets/core/secrets_collection.py

@@ -21,8 +21,15 @@ def __iter__(self) -> Generator:
 
 
 class SecretsCollection:
-    def __init__(self) -> None:
+    def __init__(self, root: str = '') -> None:
+        """
+        :param root: if specified, will scan as if the root was the value provided,
+            rather than the current working directory. We still store results as if
+            relative to root, since we're running as if it was in a different directory,
+            rather than scanning a different directory.
+        """
         self.data: Dict[str, Set[PotentialSecret]] = defaultdict(set)
+        self.root = root
 
     @classmethod
     def load_from_baseline(cls, baseline: Dict[str, Any]) -> 'SecretsCollection':
@@ -39,7 +46,7 @@ def files(self) -> Set[str]:
         return set(self.data.keys())
 
     def scan_file(self, filename: str) -> None:
-        for secret in scan.scan_file(filename):
+        for secret in scan.scan_file(os.path.join(self.root, filename)):
             self[secret.filename].add(secret)
 
     def scan_diff(self, diff: str) -> None:

detect_secrets/core/usage/__init__.py

@@ -9,6 +9,7 @@
 from . import filters
 from . import plugins
 from . import scan
+from ...settings import get_settings
 from .common import initialize_plugin_settings
 from detect_secrets.__version__ import VERSION
 
@@ -35,6 +36,17 @@ def add_default_options(self) -> 'ParserBuilder':
             version=VERSION,
             help='Display version information.',
         )
+        self._parser.add_argument(
+            '-C',
+            metavar='<path>',
+            dest='custom_root',
+            nargs=1,
+            default=[''],
+            help=(
+                'Run as if detect-secrets was started in <path>, rather than in the current '
+                'working directory.'
+            ),
+        )
 
         return self
 
@@ -137,6 +149,18 @@ def parse_args(self, argv: Optional[List[str]] = None) -> argparse.Namespace:
             print(f'error: {str(e)}', file=sys.stderr)
             sys.exit(1)
 
+        args.custom_root = args.custom_root[0]
+        if args.custom_root:
+            # This filter assumes current working directory, which will fail if we're running
+            # from a different directory.
+            # TODO: Maybe adjust this so that it is directory agnostic?
+            get_settings().disable_filters('detect_secrets.filters.common.is_invalid_file')
+
+            # Abide by the Principle of Least Surprise, and have the default value be the
+            # custom root directory itself.
+            if args.path == ['.']:
+                args.path = [args.custom_root]
+
         return args
 
 

detect_secrets/core/usage/scan.py

@@ -49,7 +49,7 @@ def _add_initialize_baseline_options(parser: argparse.ArgumentParser) -> None:
     parser.add_argument(
         'path',
         nargs='*',
-        default='.',
+        default=['.'],
         help=(
             'Scans the entire codebase and outputs a snapshot of '
             'currently identified secrets.'

detect_secrets/main.py

@@ -55,15 +55,23 @@ def handle_scan_action(args: argparse.Namespace) -> None:
         return
 
     if args.only_allowlisted:
-        secrets = SecretsCollection()
-        for filename in get_files_to_scan(*args.path, should_scan_all_files=args.all_files):
+        secrets = SecretsCollection(root=args.custom_root)
+        for filename in get_files_to_scan(
+            *args.path,
+            should_scan_all_files=args.all_files,
+            root=args.custom_root,
+        ):
             for secret in scan_for_allowlisted_secrets_in_file(filename):
                 secrets[secret.filename].add(secret)
 
         print(json.dumps(baseline.format_for_output(secrets), indent=2))
         return
 
-    secrets = baseline.create(*args.path, should_scan_all_files=args.all_files)
+    secrets = baseline.create(
+        *args.path,
+        should_scan_all_files=args.all_files,
+        root=args.custom_root,
+    )
     if args.baseline is not None:
         # The pre-commit hook's baseline upgrade is to trim the supplied baseline for non-existent
         # secrets, and to upgrade the format to the latest version. This is because the pre-commit

detect_secrets/util/git.py

@@ -3,16 +3,19 @@
 from typing import Set
 
 from ..core.log import log
-from .path import get_relative_path_if_in_cwd
+from .path import get_relative_path
 
 
-def get_root_directory() -> str:
+def get_root_directory(path: str = '') -> str:
     """
     :raises: CalledProcessError
     """
-    return subprocess.check_output(
-        'git rev-parse --show-toplevel'.split(),
-    ).decode('utf-8').strip()
+    command = ['git']
+    if path:
+        command.extend(['-C', path])
+
+    command.extend(['rev-parse', '--show-toplevel'])
+    return subprocess.check_output(command).decode('utf-8').strip()
 
 
 def get_tracked_files(root: str) -> Set[str]:
@@ -33,7 +36,7 @@ def get_tracked_files(root: str) -> Set[str]:
         )
 
         for filename in files.decode('utf-8').splitlines():
-            path = get_relative_path_if_in_cwd(os.path.join(root, filename))
+            path = get_relative_path(root, os.path.join(root, filename))
             if path:
                 output.add(path)
 

detect_secrets/util/path.py

@@ -1,7 +1,15 @@
 import os
+from pathlib import Path
 from typing import Optional
 
 
+def get_relative_path(root: str, path: str) -> Optional[str]:
+    if Path(os.getcwd()) == Path(root):
+        return get_relative_path_if_in_cwd(path)
+
+    return os.path.realpath(path)[len(root + '/'):]
+
+
 def get_relative_path_if_in_cwd(path: str) -> Optional[str]:
     filepath = os.path.realpath(path)[len(os.getcwd() + '/'):]
     if os.path.isfile(filepath):

tests/main_test.py

@@ -1,4 +1,6 @@
 import json
+import os
+import subprocess
 import tempfile
 from contextlib import contextmanager
 from contextlib import redirect_stdout
@@ -55,6 +57,20 @@ def test_saves_to_baseline():
             ]
             assert not printer.message
 
+    @staticmethod
+    def test_works_from_different_directory():
+        with tempfile.TemporaryDirectory() as d:
+            subprocess.call(['git', '-C', d, 'init'])
+            with open(os.path.join(d, 'credentials.yaml'), 'w') as f:
+                f.write('secret: asxeqFLAGMEfxuwma!')
+            subprocess.check_output(['git', '-C', d, 'add', 'credentials.yaml'])
+
+            with mock_printer(main_module) as printer:
+                assert main_module.main(['-C', d, 'scan']) == 0
+
+            results = json.loads(printer.message)['results']
+            assert results
+
 
 class TestSlimScan:
     @staticmethod