Use Python Black to format the code

.github/CONTRIBUTING.md

@@ -89,9 +89,12 @@ We try to comply with the some guidelines for new code:
     code (even if respecting Pylint rules is sometimes either too hard
     or even undesirable; human brain needed!).
 
--   [flake8](http://flake8.pycqa.org/) and Pylint tests are run for
-    each pull request (see .travis.yml for the specific options). Pull
-    requests will not be accepted when the tests fail.
+-   [Black](https://github.com/psf/black) has to be used to format
+    Python code in IVRE.
+
+-   [flake8](http://flake8.pycqa.org/), Black and Pylint tests are run
+    for each pull request (see .travis.yml for the specific
+    options). Pull requests will not be accepted when the tests fail.
 
 -   [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html)
     is a nice read!

.travis.yml

@@ -95,7 +95,7 @@ before_script:
   - export ZEEK_SAMPLES=`pwd`/usr/local/zeek/testing
   - ivre --version; echo; zeek --version; echo; nmap --version
 
-script: (test "$DB" != "maxmind" || (flake8 --ignore=F401,E402 ./doc/conf.py && flake8 --ignore=W504 setup.py ./bin/ivre && flake8 --ignore=E402,W504 ./tests/tests.py && flake8 --ignore=W504 ./ivre.bak/ && echo "flake8 OK (except W504)")) && (test "$DB" != "maxmind" || test "$TRAVIS_PYTHON_VERSION" != 3.9 || (codespell  --ignore-words=.travis/codespell_ignore `git ls-files | grep -vE '^web/static/(doc|an|bs|d3|jq|lk)/|^data/|\.(png|gif|svg)$'` && echo "codespell OK")) && (test "$DB" != "maxmind" || test "$TRAVIS_PYTHON_VERSION" != 3.9 || (pylint -e all -d abstract-method,arguments-differ,attribute-defined-outside-init,broad-except,duplicate-code,fixme,function-redefined,global-statement,global-variable-undefined,import-error,invalid-name,locally-disabled,missing-docstring,no-absolute-import,no-member,protected-access,raise-missing-from,subprocess-popen-preexec-fn,super-init-not-called,suppressed-message,too-few-public-methods,too-many-ancestors,too-many-arguments,too-many-boolean-expressions,too-many-branches,too-many-instance-attributes,too-many-lines,too-many-locals,too-many-nested-blocks,too-many-public-methods,too-many-return-statements,too-many-statements,unsubscriptable-object,unused-argument ivre && echo "pylint OK")) && cd tests/ && coverage erase && coverage run --parallel-mode tests.py --coverage && coverage combine && coverage report -i
+script: (test "$DB" != "maxmind" || test "$TRAVIS_PYTHON_VERSION" != 3.9 || (black --check ./doc/conf.py ./setup.py ./bin/ivre ./tests/tests.py ./ivre.bak/ && echo "black OK")) && (test "$DB" != "maxmind" || (flake8 --ignore=E402,E501,F401 ./doc/conf.py && flake8 --ignore=E501,W503 ./setup.py ./bin/ivre && flake8 --ignore=E203,E402,E501,W503 ./tests/tests.py && flake8 --ignore=E203,E501,W503 ./ivre.bak/ && echo "flake8 OK")) && (test "$DB" != "maxmind" || test "$TRAVIS_PYTHON_VERSION" != 3.9 || (codespell  --ignore-words=.travis/codespell_ignore `git ls-files | grep -vE '^web/static/(doc|an|bs|d3|jq|lk)/|^data/|\.(png|gif|svg)$'` && echo "codespell OK")) && (test "$DB" != "maxmind" || test "$TRAVIS_PYTHON_VERSION" != 3.9 || (pylint -e all -d abstract-method,arguments-differ,attribute-defined-outside-init,broad-except,duplicate-code,fixme,function-redefined,global-statement,global-variable-undefined,import-error,invalid-name,locally-disabled,missing-docstring,no-absolute-import,no-member,protected-access,raise-missing-from,subprocess-popen-preexec-fn,super-init-not-called,suppressed-message,too-few-public-methods,too-many-ancestors,too-many-arguments,too-many-boolean-expressions,too-many-branches,too-many-instance-attributes,too-many-lines,too-many-locals,too-many-nested-blocks,too-many-public-methods,too-many-return-statements,too-many-statements,unsubscriptable-object,unused-argument,line-too-long ivre ./doc/conf.py ./setup.py ./bin/ivre && echo "pylint OK")) && cd tests/ && coverage erase && coverage run --parallel-mode tests.py --coverage && coverage combine && coverage report -i
 
 after_success:
   - codecov

.travis/codespell_ignore

@@ -9,3 +9,4 @@ referer
 keyserver
 whos
 tha
+nin

README.md

@@ -114,7 +114,7 @@ If you are using IVRE in you research, please cite it as follows:
 Pierre Lalet, Florent Monjalet, Camille Mougey, Vincent Ruello and
 Vivien Venuti. *IVRE, a network recon framework*.
 [https://github.com/cea-sec/ivre](https://github.com/cea-sec/ivre),
-2011-2020.
+2011-2021.
 
 Here is the appropriate bibtex entry:
 
@@ -124,7 +124,7 @@ Here is the appropriate bibtex entry:
         url = {https://ivre.rocks/},
         howpublished = {\url{https://github.com/cea-sec/ivre/}},
         institution = {{CEA}: the French Alternative Energies and Atomic Energy Commission},
-        year = {2011--2020},
+        year = {2011--2021},
     }
 
 ### Technical documents & blog posts ###

bin/ivre

@@ -30,13 +30,17 @@ warnings.filterwarnings("ignore", category=DeprecationWarning)
 # because some "dev" of the cryptography module decided that
 # CryptographyDeprecationWarning should **not** inherit from
 # DeprecationWarning
-warnings.filterwarnings("ignore", message='^Python [0-9\\.]+ .*support',
-                        module='cryptography|OpenSSL')
+warnings.filterwarnings(
+    "ignore", message="^Python [0-9\\.]+ .*support", module="cryptography|OpenSSL"
+)
 
 
+# pylint: disable=wrong-import-position,cyclic-import
 from ivre import utils, tools  # noqa: E402
 from ivre.tools.version import main as version  # noqa: E402
 
+# pylint: enable=wrong-import-position,cyclic-import
+
 
 HELP_COMMANDS = ["-h", "--help", "h", "help"]
 VERSION_COMMANDS = ["-v", "--version"]
@@ -48,9 +52,11 @@ def main():
         # hack for blackarch package
         executable = executable[5:]
     if executable in tools.__all__ or executable in tools.ALIASES:
-        utils.LOGGER.warning("command %s deprecated. Use 'ivre %s' instead.",
-                             executable,
-                             tools.ALIASES.get(executable, executable))
+        utils.LOGGER.warning(
+            "command %s deprecated. Use 'ivre %s' instead.",
+            executable,
+            tools.ALIASES.get(executable, executable),
+        )
         command = tools.ALIASES.get(executable, executable)
     elif len(sys.argv) == 1:
         command = "help"
@@ -59,8 +65,7 @@ def main():
         sys.argv = ["%s %s" % (executable, sys.argv[1])] + sys.argv[2:]
     if command.lower() in HELP_COMMANDS and len(sys.argv) > 1:
         command = sys.argv[1]
-        sys.argv = ["%s %s" % (executable, sys.argv[1]),
-                    "--help"] + sys.argv[2:]
+        sys.argv = ["%s %s" % (executable, sys.argv[1]), "--help"] + sys.argv[2:]
     possible_commands = tools.guess_command(command)
     if len(possible_commands) == 1:
         tools.get_command(next(iter(possible_commands)))()
@@ -74,16 +79,19 @@ def main():
             retcode = 0
         else:
             output = sys.stderr
-            output.write("%s command: %s\n\n" % (
-                "Ambiguous" if possible_commands else "Unknown", command
-            ))
+            output.write(
+                "%s command: %s\n\n"
+                % ("Ambiguous" if possible_commands else "Unknown", command)
+            )
             retcode = 1
         version()
         output.write("usage: %s [COMMAND]\n\n" % executable)
-        output.write("%s commands:\n" % ("matching" if possible_commands
-                                         else "available"))
-        for availcmd in sorted(possible_commands if possible_commands
-                               else tools.__all__):
+        output.write(
+            "%s commands:\n" % ("matching" if possible_commands else "available")
+        )
+        for availcmd in sorted(
+            possible_commands if possible_commands else tools.__all__
+        ):
             output.write("  %s\n" % availcmd)
         output.write("\n")
         output.write("Try %s help [COMMAND]\n\n" % executable)

doc/conf.py

@@ -11,23 +11,27 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 import os
-import re
 import sys
-sys.path.insert(0, os.path.abspath('..'))
+
+sys.path.insert(0, os.path.abspath(".."))
 
 
 # -- Project information -----------------------------------------------------
 
-project = 'IVRE'
-copyright = '2011 - 2020, Pierre LALET'
-author = 'Pierre LALET'
-html_logo = '../web/static/logo.png'
-master_doc = 'index'
+# pylint: disable=no-name-in-module,wrong-import-position,redefined-builtin
+project = "IVRE"
+copyright = "2011 - 2021, Pierre LALET"
+author = "Pierre LALET"
+html_logo = "../web/static/logo.png"
+master_doc = "index"
 
 from ivre import __version__ as version
-version = version.split('dev')[0]
-if version.endswith('.'):
-    version += 'dev'
+
+# pylint: enable=no-name-in-module,wrong-import-position,redefined-builtin
+
+version = version.split("dev")[0]
+if version.endswith("."):
+    version += "dev"
 
 # -- General configuration ---------------------------------------------------
 
@@ -36,30 +40,30 @@
 # ones.
 extensions = [
     # 'sphinx.ext.autodoc', # TODO
-    'sphinx.ext.autosectionlabel',
-    'sphinx.ext.graphviz',
+    "sphinx.ext.autosectionlabel",
+    "sphinx.ext.graphviz",
     # 'sphinx.ext.napoleon', # TODO
-    'sphinxcontrib.autohttp.bottle',
+    "sphinxcontrib.autohttp.bottle",
 ]
 
 autosectionlabel_prefix_document = True
 
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
 
 # -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinx_rtd_theme'
+html_theme = "sphinx_rtd_theme"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,

doc/dev/linting.rst

@@ -6,10 +6,10 @@ mistakes and to enforce a consistent code style (or at least, attempt
 to do so).
 
 So far, only the Python code uses such linters (`Flake8
-<https://flake8.pycqa.org>`_ and `Pylint
-<https://www.pylint.org/>`_). Adding similar code linting capabilities
-to the Zeek scripts (`zeek/`), LUA (`nmap_scripts/`) and JavaScript /
-HTML (`web/static/`) could be a good PR idea!
+<https://flake8.pycqa.org>`_, `Pylint <https://www.pylint.org/>`_ and
+`Black <https://github.com/psf/black>`_). Adding similar code linting
+capabilities to the Zeek scripts (`zeek/`), LUA (`nmap_scripts/`) and
+JavaScript / HTML (`web/static/`) could be a good PR idea!
 
 For all the code and the documentation, we also use `Codespell
 <https://github.com/codespell-project/codespell>`_ to prevent typos.
@@ -19,8 +19,8 @@ Running the linters
 
 To install the Python code linters and Codespell you can simply use
 the `requirements-linting.txt` file with Pip, or use any method to
-install the latest versions of the `codespell`, `flake8` and `pylint`
-Python modules.
+install the latest versions of the `black`, `codespell`, `flake8` and
+`pylint` Python modules.
 
 The script `pkg/runchecks` will run all the tests for you with the
 expected options and exceptions.

ivre/__init__.py

@@ -29,7 +29,7 @@
 
 
 _DIR = os.path.dirname(__file__)
-_VERSION_FILE = os.path.join(_DIR, 'VERSION')
+_VERSION_FILE = os.path.join(_DIR, "VERSION")
 
 
 def _get_version_from_file():
@@ -41,29 +41,35 @@ def _get_version_from_file():
 
 
 def _get_version_from_git():
-    proc = subprocess.Popen([b'git', b'rev-parse', b'--show-toplevel'],
-                            stdout=subprocess.PIPE, stderr=open(os.devnull),
-                            cwd=os.path.join(_DIR, os.path.pardir))
+    proc = subprocess.Popen(
+        [b"git", b"rev-parse", b"--show-toplevel"],
+        stdout=subprocess.PIPE,
+        stderr=open(os.devnull),
+        cwd=os.path.join(_DIR, os.path.pardir),
+    )
     out, err = proc.communicate()
     if proc.returncode != 0:
         raise subprocess.CalledProcessError(proc.returncode, err)
     repo = out.decode().strip()
     if repo != os.path.realpath(os.path.join(_DIR, os.path.pardir)):
         raise ValueError("Git repository is not IVRE")
-    proc = subprocess.Popen([b'git', b'describe', b'--always'],
-                            stdout=subprocess.PIPE, stderr=open(os.devnull),
-                            cwd=os.path.join(_DIR, os.path.pardir))
+    proc = subprocess.Popen(
+        [b"git", b"describe", b"--always"],
+        stdout=subprocess.PIPE,
+        stderr=open(os.devnull),
+        cwd=os.path.join(_DIR, os.path.pardir),
+    )
     out, err = proc.communicate()
     if proc.returncode != 0:
         raise subprocess.CalledProcessError(proc.returncode, err)
     tag = out.decode().strip()
-    match = re.match('^v?(.+?)-(\\d+)-g[a-f0-9]+$', tag)
+    match = re.match("^v?(.+?)-(\\d+)-g[a-f0-9]+$", tag)
     if match:
         # remove the 'v' prefix and add a '.devN' suffix
-        value = '%s.dev%s' % match.groups()
+        value = "%s.dev%s" % match.groups()
     else:
         # just remove the 'v' prefix
-        value = tag[1:] if tag.startswith('v') else tag
+        value = tag[1:] if tag.startswith("v") else tag
     return value
 
 
@@ -74,7 +80,7 @@ def _version():
         pass
     else:
         try:
-            with open(_VERSION_FILE, 'w') as fdesc:
+            with open(_VERSION_FILE, "w") as fdesc:
                 fdesc.write(tag)
         except IOError:
             pass
@@ -84,15 +90,14 @@ def _version():
             return fdesc.read()
     except IOError:
         pass
-    hashval, refnames = '$Format:%h %D$'.split(' ', 1)
+    hashval, refnames = "$Format:%h %D$".split(" ", 1)
     try:
-        return next(ref[6:] for ref in refnames.split(', ')
-                    if ref.startswith('tag: v'))
+        return next(ref[6:] for ref in refnames.split(", ") if ref.startswith("tag: v"))
     except StopIteration:
         pass
-    if hashval == '$Format:%h':
-        return 'unknown.version'
-    return hashval if hashval else 'unknown.version'
+    if hashval == "$Format:%h":
+        return "unknown.version"
+    return hashval if hashval else "unknown.version"
 
 
 __version__ = VERSION = _version()

ivre/active/cpe.py

@@ -57,13 +57,13 @@ def cpe2dict(cpe_str):
 def add_cpe_values(hostrec, path, cpe_values):
     """Add CPE values (`cpe_values`) to the `hostrec` at the given `path`.
 
-CPEs are indexed in a dictionary to agglomerate origins, but this dict
-is replaced with its values() in ._pre_addhost() or in
-.store_scan_json_zgrab(), or in the function that calls
-add_cpe_values(), depending on the context.
+    CPEs are indexed in a dictionary to agglomerate origins, but this dict
+    is replaced with its values() in ._pre_addhost() or in
+    .store_scan_json_zgrab(), or in the function that calls
+    add_cpe_values(), depending on the context.
 
     """
-    cpes = hostrec.setdefault('cpes', {})
+    cpes = hostrec.setdefault("cpes", {})
     for cpe in cpe_values:
         if cpe not in cpes:
             try:
@@ -74,4 +74,4 @@ def add_cpe_values(hostrec, path, cpe_values):
             cpes[cpe] = cpeobj
         else:
             cpeobj = cpes[cpe]
-        cpeobj.setdefault('origins', set()).add(path)
+        cpeobj.setdefault("origins", set()).add(path)