Bug 1555560: Add support for black formatting with mozlint; r=sylvestre

Differential Revision: https://phabricator.services.mozilla.com/D33125

.flake8

@@ -96,6 +96,10 @@ ignore =
     #       text contains quotes which breaks our custom JSON formatter
     F723, E121, E123, E126, E129, E133, E226, E241, E242, E402, E704, E741, W503,
 
+    # black will generate code that breaks these. they're not PEP8 compliant though;
+    # see https://github.com/psf/black/blob/master/docs/compatible_configs.md#flake8
+    W503, E203
+
 per-file-ignores =
     # These paths are intentionally excluded.
     ipc/ipdl/*: F403, F405

docs/code-quality/index.rst

@@ -100,6 +100,11 @@ In this document, we try to list these all tools.
      - `bug 1155970 <https://bugzilla.mozilla.org/show_bug.cgi?id=1155970>`_
      - :ref:`Flake8`
      - http://flake8.pycqa.org/
+   * - black
+     - Yes
+     - `bug 1555560 <https://bugzilla.mozilla.org/show_bug.cgi?id=1555560>`_
+     - :ref:`black`
+     - https://black.readthedocs.io/en/stable
    * - pylint
      -
      - `bug 1623024 <https://bugzilla.mozilla.org/show_bug.cgi?id=1623024>`_

docs/code-quality/lint/linters/black.rst

@@ -0,0 +1,36 @@
+Black
+=====
+
+`Black <https://black.readthedocs.io/en/stable/>`__ is a opinionated python code formatter.
+
+
+Run Locally
+-----------
+
+The mozlint integration of black can be run using mach:
+
+.. parsed-literal::
+
+    $ mach lint --linter black <file paths>
+
+Alternatively, omit the ``--linter black`` and run all configured linters, which will include
+black.
+
+
+Configuration
+-------------
+
+To enable black on new directory, add the path to the include
+section in the `black.yml <https://searchfox.org/mozilla-central/source/tools/lint/black.yml>`_ file.
+
+Autofix
+-------
+
+The black linter provides a ``--fix`` option.
+
+
+Sources
+-------
+
+* `Configuration (YAML) <https://searchfox.org/mozilla-central/source/tools/lint/black.yml>`_
+* `Source <https://searchfox.org/mozilla-central/source/tools/lint/python/black.py>`_

testing/condprofile/condprof/util.py

@@ -223,7 +223,8 @@ def download_file(url, target=None):
         try:
             archivedir = os.path.dirname(target)
             logger.info(
-                "Content in cache directory %s: %s" % (archivedir, os.listdir(archivedir))
+                "Content in cache directory %s: %s"
+                % (archivedir, os.listdir(archivedir))
             )
         except Exception:
             logger.info("Failed to list cache directory contents")

tools/lint/black.yml

@@ -0,0 +1,22 @@
+---
+black:
+    description: Reformat python
+    include:
+        - python/mozperftest/mozperftest
+        - taskcluster/docker/funsize-update-generator
+        - testing/condprofile/condprof
+        - tools/crashreporter/system-symbols
+        - tools/lint/python/black.py
+        - tools/lint/test/test_black.py
+        - tools/tryselect/selectors/scriptworker.py
+    exclude:
+        - layout/style/ServoCSSPropList.mako.py
+        - testing/mozharness/configs/test/test_malformed.py
+        - tools/lint/test/files
+    extensions:
+        - py
+    support-files:
+        - 'tools/lint/python/**'
+    type: external
+    payload: python.black:lint
+    setup: python.black:setup

tools/lint/python/black.py

@@ -0,0 +1,129 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function
+
+import os
+import platform
+import re
+import signal
+import subprocess
+import sys
+
+from mozlint import result
+from mozlint.pathutils import expand_exclusions
+from mozlint.util import pip
+from mozprocess import ProcessHandler
+
+here = os.path.abspath(os.path.dirname(__file__))
+BLACK_REQUIREMENTS_PATH = os.path.join(here, "black_requirements.txt")
+
+BLACK_INSTALL_ERROR = """
+Unable to install correct version of black
+Try to install it manually with:
+    $ pip install -U --require-hashes -r {}
+""".strip().format(
+    BLACK_REQUIREMENTS_PATH
+)
+
+# We use sys.prefix to find executables as that gets modified with
+# virtualenv's activate_this.py, whereas sys.executable doesn't.
+if platform.system() == "Windows":
+    bindir = os.path.join(sys.prefix, "Scripts")
+else:
+    bindir = os.path.join(sys.prefix, "bin")
+
+
+def get_black_version(binary):
+    """
+    Returns found binary's version
+    """
+    try:
+        output = subprocess.check_output(
+            [binary, "--version"], stderr=subprocess.STDOUT, universal_newlines=True,
+        )
+    except subprocess.CalledProcessError as e:
+        output = e.output
+
+    return re.match(r"black, version (.*)$", output)[1]
+
+
+def parse_issues(config, output, paths, *, log):
+    would_reformat = re.compile("^would reformat (.*)$", re.I)
+    reformatted = re.compile("^reformatted (.*)$", re.I)
+    cannot_reformat = re.compile("^error: cannot format (.*?): (.*)$", re.I)
+    results = []
+    for line in output:
+        line = line.decode("utf-8")
+        if line.startswith("All done!") or line.startswith("Oh no!"):
+            break
+
+        match = would_reformat.match(line)
+        if match:
+            res = {"path": match.group(1), "level": "error"}
+            results.append(result.from_config(config, **res))
+            continue
+
+        match = reformatted.match(line)
+        if match:
+            res = {"path": match.group(1), "level": "warning", "message": "reformatted"}
+            results.append(result.from_config(config, **res))
+            continue
+
+        match = cannot_reformat.match(line)
+        if match:
+            res = {"path": match.group(1), "level": "error", "message": match.group(2)}
+            results.append(result.from_config(config, **res))
+            continue
+
+        log.debug("Unhandled line", line)
+    return results
+
+
+class BlackProcess(ProcessHandler):
+    def __init__(self, config, *args, **kwargs):
+        self.config = config
+        kwargs["stream"] = False
+        ProcessHandler.__init__(self, *args, **kwargs)
+
+    def run(self, *args, **kwargs):
+        orig = signal.signal(signal.SIGINT, signal.SIG_IGN)
+        ProcessHandler.run(self, *args, **kwargs)
+        signal.signal(signal.SIGINT, orig)
+
+
+def run_process(config, cmd):
+    proc = BlackProcess(config, cmd)
+    proc.run()
+    try:
+        proc.wait()
+    except KeyboardInterrupt:
+        proc.kill()
+
+    return proc.output
+
+
+def setup(root, **lintargs):
+    if not pip.reinstall_program(BLACK_REQUIREMENTS_PATH):
+        print(BLACK_INSTALL_ERROR)
+        return 1
+
+
+def run_black(config, paths, fix=None, *, log):
+    binary = os.path.join(bindir, "black")
+
+    log.debug("Black version {}".format(get_black_version(binary)))
+
+    cmd_args = [binary]
+    if not fix:
+        cmd_args.append("--check")
+    base_command = cmd_args + paths
+    log.debug("Command: {}".format(" ".join(base_command)))
+    return parse_issues(config, run_process(config, base_command), paths, log=log)
+
+
+def lint(paths, config, fix=None, **lintargs):
+    files = list(expand_exclusions(paths, config, lintargs["root"]))
+
+    return run_black(config, files, fix=fix, log=lintargs["log"])

tools/lint/python/black_requirements.in

@@ -0,0 +1 @@
+black==19.10b0

tools/lint/python/black_requirements.txt

@@ -0,0 +1,75 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --generate-hashes --output-file=tools/lint/python/black_requirements.txt tools/lint/python/black_requirements.in
+#
+appdirs==1.4.3 \
+    --hash=sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92 \
+    --hash=sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e \
+    # via black
+attrs==19.1.0 \
+    --hash=sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79 \
+    --hash=sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399 \
+    # via black
+black==19.10b0 \
+    --hash=sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b \
+    --hash=sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539 \
+    # via -r tools/lint/python/black_requirements.in
+click==7.0 \
+    --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \
+    --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \
+    # via black
+pathspec==0.7.0 \
+    --hash=sha256:163b0632d4e31cef212976cf57b43d9fd6b0bac6e67c26015d611a647d5e7424 \
+    --hash=sha256:562aa70af2e0d434367d9790ad37aed893de47f1693e4201fd1d3dca15d19b96 \
+    # via black
+regex==2020.1.8 \
+    --hash=sha256:07b39bf943d3d2fe63d46281d8504f8df0ff3fe4c57e13d1656737950e53e525 \
+    --hash=sha256:0932941cdfb3afcbc26cc3bcf7c3f3d73d5a9b9c56955d432dbf8bbc147d4c5b \
+    --hash=sha256:0e182d2f097ea8549a249040922fa2b92ae28be4be4895933e369a525ba36576 \
+    --hash=sha256:10671601ee06cf4dc1bc0b4805309040bb34c9af423c12c379c83d7895622bb5 \
+    --hash=sha256:23e2c2c0ff50f44877f64780b815b8fd2e003cda9ce817a7fd00dea5600c84a0 \
+    --hash=sha256:26ff99c980f53b3191d8931b199b29d6787c059f2e029b2b0c694343b1708c35 \
+    --hash=sha256:27429b8d74ba683484a06b260b7bb00f312e7c757792628ea251afdbf1434003 \
+    --hash=sha256:3e77409b678b21a056415da3a56abfd7c3ad03da71f3051bbcdb68cf44d3c34d \
+    --hash=sha256:4e8f02d3d72ca94efc8396f8036c0d3bcc812aefc28ec70f35bb888c74a25161 \
+    --hash=sha256:4eae742636aec40cf7ab98171ab9400393360b97e8f9da67b1867a9ee0889b26 \
+    --hash=sha256:6a6ae17bf8f2d82d1e8858a47757ce389b880083c4ff2498dba17c56e6c103b9 \
+    --hash=sha256:6a6ba91b94427cd49cd27764679024b14a96874e0dc638ae6bdd4b1a3ce97be1 \
+    --hash=sha256:7bcd322935377abcc79bfe5b63c44abd0b29387f267791d566bbb566edfdd146 \
+    --hash=sha256:98b8ed7bb2155e2cbb8b76f627b2fd12cf4b22ab6e14873e8641f266e0fb6d8f \
+    --hash=sha256:bd25bb7980917e4e70ccccd7e3b5740614f1c408a642c245019cff9d7d1b6149 \
+    --hash=sha256:d0f424328f9822b0323b3b6f2e4b9c90960b24743d220763c7f07071e0778351 \
+    --hash=sha256:d58e4606da2a41659c84baeb3cfa2e4c87a74cec89a1e7c56bee4b956f9d7461 \
+    --hash=sha256:e3cd21cc2840ca67de0bbe4071f79f031c81418deb544ceda93ad75ca1ee9f7b \
+    --hash=sha256:e6c02171d62ed6972ca8631f6f34fa3281d51db8b326ee397b9c83093a6b7242 \
+    --hash=sha256:e7c7661f7276507bce416eaae22040fd91ca471b5b33c13f8ff21137ed6f248c \
+    --hash=sha256:ecc6de77df3ef68fee966bb8cb4e067e84d4d1f397d0ef6fce46913663540d77 \
+    # via black
+toml==0.10.0 \
+    --hash=sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c \
+    --hash=sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e \
+    # via black
+typed-ast==1.4.0 \
+    --hash=sha256:1170afa46a3799e18b4c977777ce137bb53c7485379d9706af8a59f2ea1aa161 \
+    --hash=sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e \
+    --hash=sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e \
+    --hash=sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0 \
+    --hash=sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c \
+    --hash=sha256:48e5b1e71f25cfdef98b013263a88d7145879fbb2d5185f2a0c79fa7ebbeae47 \
+    --hash=sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631 \
+    --hash=sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4 \
+    --hash=sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34 \
+    --hash=sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b \
+    --hash=sha256:7954560051331d003b4e2b3eb822d9dd2e376fa4f6d98fee32f452f52dd6ebb2 \
+    --hash=sha256:838997f4310012cf2e1ad3803bce2f3402e9ffb71ded61b5ee22617b3a7f6b6e \
+    --hash=sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a \
+    --hash=sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233 \
+    --hash=sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1 \
+    --hash=sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36 \
+    --hash=sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d \
+    --hash=sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a \
+    --hash=sha256:fdc1c9bbf79510b76408840e009ed65958feba92a88833cdceecff93ae8fff66 \
+    --hash=sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12 \
+    # via black

tools/lint/test/files/black/bad.py

@@ -0,0 +1,6 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+print (
+    "test"
+       )

tools/lint/test/files/black/invalid.py

@@ -0,0 +1,4 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+print(

tools/lint/test/python.ini

@@ -2,6 +2,8 @@
 subsuite = mozlint
 skip-if = python == 2
 
+[test_black.py]
+requirements = tools/lint/python/black_requirements.txt
 [test_eslint.py]
 [test_flake8.py]
 requirements = tools/lint/python/flake8_requirements.txt

tools/lint/test/test_black.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import mozunit
+
+LINTER = "black"
+
+
+def test_lint_black(lint, paths):
+    results = lint(paths())
+    assert len(results) == 2
+
+    assert "EOF" in results[0].message
+    assert results[0].level == "error"
+    assert results[0].relpath == "invalid.py"
+
+    assert results[1].level == "error"
+    assert results[1].relpath == "bad.py"
+
+
+if __name__ == "__main__":
+    mozunit.main()