add archivebox config command and move config into sections

archivebox/cli/archivebox_config.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox config'
+__description__ = 'Get and set your ArchiveBox project configuration values'
+
+import sys
+import argparse
+
+from typing import Optional, List
+
+from ..legacy.util import SmartFormatter
+from ..legacy.config import (
+    check_data_folder,
+    OUTPUT_DIR,
+    write_config_file,
+    CONFIG,
+    ConfigDict,
+    stderr,
+)
+
+
+def main(args: List[str]=None, stdin: Optional[str]=None) -> None:
+    check_data_folder()
+
+    args = sys.argv[1:] if args is None else args
+
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=__description__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        '--get', #'-g',
+        action='store_true',
+        help="Get the value for the given config KEYs",
+    )
+    group.add_argument(
+        '--set', #'-s',
+        action='store_true',
+        help="Set the given KEY=VALUE config values",
+    )
+    parser.add_argument(
+        'config_options',
+        nargs='*',
+        type=str,
+        help='KEY or KEY=VALUE formatted config values to get or set',
+    )
+    command = parser.parse_args(args)
+
+    if stdin or not sys.stdin.isatty():
+        stdin_raw_text = stdin or sys.stdin.read()
+        if stdin_raw_text and command.config_options:
+            stderr(
+                '[X] You should either pass config values as an arguments '
+                'or via stdin, but not both.\n',
+                color='red',
+            )
+            raise SystemExit(1)
+
+        config_options = stdin_raw_text.split('\n')
+    else:
+        config_options = command.config_options
+
+    no_args = not (command.get or command.set or command.config_options)
+
+    matching_config: ConfigDict = {}
+    if command.get or no_args:
+        if config_options:
+            matching_config = {key: CONFIG[key] for key in config_options if key in CONFIG}
+            failed_config = [key for key in config_options if key not in CONFIG]
+            if failed_config:
+                stderr()
+                stderr('[X] These options failed to get', color='red')
+                stderr('    {}'.format('\n    '.join(config_options)))
+                raise SystemExit(1)
+        else:
+            matching_config = CONFIG
+
+        print('\n'.join(f'{key}={val}' for key, val in matching_config.items()))
+        raise SystemExit(not matching_config)
+    elif command.set:
+        new_config = {}
+        failed_options = []
+        for line in config_options:
+            if line.startswith('#') or not line.strip():
+                continue
+            if '=' not in line:
+                stderr('[X] Config KEY=VALUE must have an = sign in it', color='red')
+                stderr(f'    {line}')
+                raise SystemExit(2)
+
+            key, val = line.split('=')
+            if key.upper().strip() in CONFIG:
+                new_config[key.upper().strip()] = val.strip()
+            else:
+                failed_options.append(line)
+
+        if new_config:
+            matching_config = write_config_file(new_config, out_dir=OUTPUT_DIR)
+            print('\n'.join(f'{key}={val}' for key, val in matching_config.items()))
+        if failed_options:
+            stderr()
+            stderr('[X] These options failed to set:', color='red')
+            stderr('    {}'.format('\n    '.join(failed_options)))
+        raise SystemExit(bool(failed_options))
+    else:
+        stderr('[X] You must pass either --get or --set, or no arguments to get the whole config.', color='red')
+        stderr('    archivebox config')
+        stderr('    archivebox config --get SOME_KEY')
+        stderr('    archivebox config --set SOME_KEY=SOME_VALUE')
+        raise SystemExit(2)
+
+if __name__ == '__main__':
+    main()

archivebox/legacy/ArchiveBox.conf

@@ -0,0 +1,58 @@
+# This is the example default configiration file for ArchiveBox.
+# 
+# Copy example config from here into your project's ArchiveBox.conf file,
+# DO NOT EDIT THIS FILE DIRECTLY!
+#
+# See the list of all the possible options. documentation, and examples here:
+#    https://github.com/pirate/ArchiveBox/wiki/Configuration
+
+[GENERAL_CONFIG]
+OUTPUT_PERMISSIONS = 755
+ONLY_NEW = False
+TIMEOUT = 60
+MEDIA_TIMEOUT = 3600
+ACTIVE_THEME = default
+FOOTER_INFO = Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests.
+URL_BLACKLIST = (://(.*\.)?facebook\.com)|(://(.*\.)?ebay\.com)|(.*\.exe$)
+
+[ARCHIVE_METHOD_TOGGLES]
+SAVE_TITLE = True
+SAVE_FAVICON = True
+SAVE_WGET = True
+SAVE_WGET_REQUISITES = True
+SAVE_WARC = True
+SAVE_PDF = True
+SAVE_SCREENSHOT = True
+SAVE_DOM = True
+SAVE_GIT = True
+SAVE_MEDIA = False
+SAVE_ARCHIVE_DOT_ORG = True
+
+
+[ARCHIVE_METHOD_OPTIONS]
+CHECK_SSL_VALIDITY = True
+RESOLUTION = 1440,900
+GIT_DOMAINS = github.com,bitbucket.org,gitlab.com
+
+CROME_HEADLESS = True
+CROME_SANDBOX = True
+
+COOKIES_FILE = path/to/cookies.txt
+CHROME_USER_DATA_DIR = ~/.config/google-chrome/Default
+
+WGET_USER_AGENT = Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36
+CHROME_USER_AGENT = Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36
+
+
+[DEPENDENCY_CONFIG]
+USE_CURL = True
+USE_WGET = True
+USE_CHROME = True
+USE_YOUTUBEDL = True
+USE_GIT = True
+
+CURL_BINARY = curl
+GIT_BINARY = git"
+WGET_BINARY = wget
+YOUTUBEDL_BINARY = youtube-dl
+CHROME_BINARY = chromium