libs: Add envoy.code_format.python_check (#88)

Signed-off-by: Ryan Northey <[email protected]>

.github/workflows/ci.yml

@@ -17,6 +17,7 @@ jobs:
         - envoy.abstract.command
         - envoy.base.runner
         - envoy.base.utils
+        - envoy.code_format.python_check
         - envoy.docker.utils
         - envoy.dependency.pip_check
         - envoy.distribution.distrotest
@@ -102,6 +103,7 @@ jobs:
         - envoy.abstract.command
         - envoy.base.runner
         - envoy.base.utils
+        - envoy.code_format.python_check
         - envoy.dependency.pip_check
         - envoy.docker.utils
         - envoy.distribution.distrotest

envoy.code_format.python_check/README.rst

@@ -0,0 +1,5 @@
+
+envoy.code_format.python_check
+==============================
+
+Python code format linter/checker used in Envoy proxy's CI

envoy.code_format.python_check/VERSION

@@ -0,0 +1 @@
+0.0.1

envoy.code_format.python_check/envoy/code_format/python_check/__init__.py

@@ -0,0 +1,13 @@
+
+from .abstract import APythonChecker
+# from .exceptions import PipConfigurationError
+from .checker import PythonChecker
+from .cmd import cmd, main
+
+
+__all__ = (
+    "APythonChecker",
+    "cmd",
+    "main",
+    "PipConfigurationError",
+    "PythonChecker")

envoy.code_format.python_check/envoy/code_format/python_check/abstract.py

@@ -0,0 +1,150 @@
+
+import abc
+import argparse
+import pathlib
+from functools import cached_property
+from typing import Iterable, List, Optional, Tuple
+
+from flake8.main.application import (  # type:ignore
+    Application as Flake8Application)
+
+import yapf  # type:ignore
+
+import abstracts
+
+from aio.subprocess import run
+from aio.tasks import concurrent
+
+from envoy.base import checker, utils
+
+
+FLAKE8_CONFIG = '.flake8'
+YAPF_CONFIG = '.style.yapf'
+
+# TODO(phlax): add checks for:
+#      - isort
+
+
+class APythonChecker(checker.AsyncChecker, metaclass=abstracts.Abstraction):
+    checks = ("flake8", "yapf")
+
+    @property
+    def diff_file_path(self) -> Optional[pathlib.Path]:
+        return (
+            pathlib.Path(self.args.diff_file)
+            if self.args.diff_file
+            else None)
+
+    @cached_property
+    def flake8_app(self) -> Flake8Application:
+        flake8_app = Flake8Application()
+        flake8_app.initialize(self.flake8_args)
+        return flake8_app
+
+    @property
+    def flake8_args(self) -> Tuple[str, ...]:
+        return ("--config", str(self.flake8_config_path), str(self.path))
+
+    @property
+    def flake8_config_path(self) -> pathlib.Path:
+        return self.path.joinpath(FLAKE8_CONFIG)
+
+    @property
+    @abc.abstractmethod
+    def path(self) -> pathlib.Path:
+        return super().path
+
+    @property
+    def recurse(self) -> bool:
+        """Flag to determine whether to apply checks recursively"""
+        return self.args.recurse
+
+    @property
+    def yapf_config_path(self) -> pathlib.Path:
+        return self.path.joinpath(YAPF_CONFIG)
+
+    @property
+    def yapf_files(self) -> List[str]:
+        return yapf.file_resources.GetCommandLineFiles(
+            self.args.paths,
+            recursive=self.recurse,
+            exclude=yapf.file_resources.GetExcludePatternsForDir(
+                str(self.path)))
+
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument(
+            "--recurse",
+            "-r",
+            choices=["yes", "no"],
+            default="yes",
+            help="Recurse path or paths directories")
+        parser.add_argument(
+            "--diff-file",
+            default=None,
+            help="Specify the path to a diff file with fixes")
+
+    async def check_flake8(self) -> None:
+        """Run flake8 on files and/or repo"""
+        errors: List[str] = []
+        with utils.buffered(stdout=errors, mangle=self._strip_lines):
+            self.flake8_app.run_checks()
+            self.flake8_app.report()
+        if errors:
+            self.error("flake8", errors)
+
+    async def check_yapf(self) -> None:
+        """Run flake8 on files and/or repo"""
+        futures = concurrent(
+            self.yapf_format(python_file)
+            for python_file
+            in self.yapf_files)
+
+        async for (python_file, (reformatted, encoding, changed)) in futures:
+            self.yapf_result(python_file, reformatted, changed)
+
+    async def on_check_run(self, check: str) -> None:
+        if check not in self.failed and check not in self.warned:
+            self.succeed(check, [check])
+
+    async def on_checks_complete(self) -> int:
+        if self.diff_file_path and self.has_failed:
+            result = await run(
+                ["git", "diff", "HEAD"],
+                cwd=self.path,
+                capture_output=True)
+            self.diff_file_path.write_bytes(result.stdout)
+        return await super().on_checks_complete()
+
+    async def yapf_format(self, python_file: str) -> tuple:
+        return python_file, yapf.yapf_api.FormatFile(
+            python_file,
+            style_config=str(self.yapf_config_path),
+            in_place=self.fix,
+            print_diff=not self.fix)
+
+    def yapf_result(
+            self,
+            python_file: str,
+            reformatted: str,
+            changed: bool) -> None:
+        if not changed:
+            return self.succeed("yapf", [python_file])
+        if self.fix:
+            return self.warn("yapf", [f"{python_file}: reformatted"])
+        if reformatted:
+            return self.warn("yapf", [f"{python_file}: diff\n{reformatted}"])
+        self.error("yapf", [python_file])
+
+    def _strip_line(self, line: str) -> str:
+        return (
+            line[len(str(self.path)) + 1:]
+            if line.startswith(f"{self.path}/")
+            else line)
+
+    def _strip_lines(self, lines: Iterable[str]) -> List[str]:
+        return [
+            self._strip_line(line)
+            for line
+            in lines
+            if line]

envoy.code_format.python_check/envoy/code_format/python_check/checker.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+
+import pathlib
+from functools import cached_property
+
+import abstracts
+
+
+from .abstract import APythonChecker
+
+
+@abstracts.implementer(APythonChecker)
+class PythonChecker:
+
+    @cached_property
+    def path(self) -> pathlib.Path:
+        return super().path

envoy.code_format.python_check/envoy/code_format/python_check/cmd.py

@@ -0,0 +1,16 @@
+
+import sys
+
+from .checker import PythonChecker
+
+
+def main(*args: str) -> int:
+    return PythonChecker(*args).run()
+
+
+def cmd():
+    sys.exit(main(*sys.argv[1:]))
+
+
+if __name__ == "__main__":
+    cmd()

envoy.code_format.python_check/mypy.ini

@@ -0,0 +1,2 @@
+[mypy]
+plugins = mypy_abstracts

envoy.code_format.python_check/setup.cfg

@@ -0,0 +1,57 @@
+[metadata]
+name = envoy.code_format.python_check
+version = file: VERSION
+author = Ryan Northey
+author_email = [email protected]
+maintainer = Ryan Northey
+maintainer_email = [email protected]
+license = Apache Software License 2.0
+url = https://github.com/envoyproxy/pytooling/tree/main/envoy.code_format.python_check
+description = "Python code format linter/checker used in Envoy proxy's CI"
+long_description = file: README.rst
+classifiers =
+    Development Status :: 4 - Beta
+    Framework :: Pytest
+    Intended Audience :: Developers
+    Topic :: Software Development :: Testing
+    Programming Language :: Python
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3 :: Only
+    Programming Language :: Python :: Implementation :: CPython
+    Operating System :: OS Independent
+    License :: OSI Approved :: Apache Software License
+
+[options]
+python_requires = >=3.5
+py_modules = envoy.code_format.python_check
+packages = find_namespace:
+install_requires =
+    abstracts>=0.0.12
+    aio.subprocess
+    aio.tasks
+    envoy.base.checker>=0.0.2
+    envoy.base.utils>=0.0.3
+    flake8
+    pep8-naming
+    yapf
+
+[options.extras_require]
+test =
+    pytest
+    pytest-asyncio
+    pytest-coverage
+    pytest-patches
+lint = flake8
+types =
+    mypy
+    mypy-abstracts
+publish = wheel
+
+[options.package_data]
+* = py.typed
+
+[options.entry_points]
+console_scripts =
+    envoy.code_format.python_check = envoy.code_format.python_check:cmd

envoy.code_format.python_check/setup.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+
+from setuptools import setup  # type:ignore
+
+setup()