Add flake8 extensions (iterative#4067)

* remove unnecessary generators/comprehensions

* Use dict literals

* remove unused variables

* misc fixes

ignore .iteritems, is a false +ve
remove function call in default argument

* Flake8 config: complexity, show source on err, show err counts

* flake8: add extensions

* remove flake8-string-format

* Add flake8-debugger support

* Add flake8-string-format extension

* remove flake8-debugger

It's having some problems to install on Windows in Travis

.pre-commit-config.yaml

@@ -1,63 +1,68 @@
 repos:
-- hooks:
-  - id: black
-    language_version: python3
-  repo: https://github.com/ambv/black
-  rev: 19.10b0
--   repo: https://github.com/asottile/seed-isort-config
+  - hooks:
+      - id: black
+        language_version: python3
+    repo: https://github.com/ambv/black
+    rev: 19.10b0
+  - repo: https://github.com/asottile/seed-isort-config
     rev: v2.1.1
     hooks:
-    -   id: seed-isort-config
-- hooks:
-  - id: isort
-    language_version: python3
-  repo: https://github.com/timothycrosley/isort
-  rev: 4.3.21
-- hooks:
-  - id: flake8
-    language_version: python3
-  repo: https://gitlab.com/pycqa/flake8
-  rev: 3.7.9
-- hooks:
-  - args:
-    - -i
-    - '2'
-    id: beautysh
-    language_version: python3
-  repo: https://github.com/lovesegfault/beautysh
-  rev: master
-- hooks:
-  - args:
-    - git-hook
-    - pre-commit
-    entry: dvc
-    id: dvc-pre-commit
-    language: system
-    name: DVC pre-commit
-    stages:
-    - commit
-    verbose: true
-    require_serial: true
-  - args:
-    - git-hook
-    - pre-push
-    entry: dvc
-    id: dvc-pre-push
-    language: system
-    name: DVC pre-push
-    stages:
-    - push
-    require_serial: true
-  - always_run: true
-    args:
-    - git-hook
-    - post-checkout
-    entry: dvc
-    id: dvc-post-checkout
-    language: system
-    minimum_pre_commit_version: 2.2.0
-    name: DVC post-checkout
-    stages:
-    - post-checkout
-    require_serial: true
-  repo: local
+      - id: seed-isort-config
+  - hooks:
+      - id: isort
+        language_version: python3
+    repo: https://github.com/timothycrosley/isort
+    rev: 4.3.21
+  - hooks:
+      - id: flake8
+        language_version: python3
+        additional_dependencies:
+          - flake8-bugbear
+          - flake8-comprehensions
+          - flake8-debugger
+          - flake8-string-format
+    repo: https://gitlab.com/pycqa/flake8
+    rev: 3.7.9
+  - hooks:
+      - args:
+          - -i
+          - "2"
+        id: beautysh
+        language_version: python3
+    repo: https://github.com/lovesegfault/beautysh
+    rev: master
+  - hooks:
+      - args:
+          - git-hook
+          - pre-commit
+        entry: dvc
+        id: dvc-pre-commit
+        language: system
+        name: DVC pre-commit
+        stages:
+          - commit
+        verbose: true
+        require_serial: true
+      - args:
+          - git-hook
+          - pre-push
+        entry: dvc
+        id: dvc-pre-push
+        language: system
+        name: DVC pre-push
+        stages:
+          - push
+        require_serial: true
+      - always_run: true
+        args:
+          - git-hook
+          - post-checkout
+        entry: dvc
+        id: dvc-post-checkout
+        language: system
+        minimum_pre_commit_version: 2.2.0
+        name: DVC post-checkout
+        stages:
+          - post-checkout
+        require_serial: true
+    repo: local

dvc/command/diff.py

@@ -19,7 +19,7 @@ def _show_md(diff):
         entries = diff.get(status, [])
         if not entries:
             continue
-        paths = sorted([entry["path"] for entry in entries])
+        paths = sorted(entry["path"] for entry in entries)
         for path in paths:
             rows.append([status, path])
 

dvc/dagascii.py

@@ -222,7 +222,7 @@ def _build_sugiyama_layout(vertexes, edges):
         vertex.view = VertexViewer(vertex.data)
 
     # NOTE: determine min box length to create the best layout
-    minw = min([v.view.w for v in vertexes])
+    minw = min(v.view.w for v in vertexes)
 
     for edge in edges:
         edge.view = EdgeViewer()

dvc/remote/ssh/connection.py

@@ -153,7 +153,7 @@ def walk(self, directory, topdown=True):
             yield directory, dirs, nondirs
 
     def walk_files(self, directory):
-        for root, dirs, files in self.walk(directory):
+        for root, _, files in self.walk(directory):
             for fname in files:
                 yield posixpath.join(root, fname)
 

dvc/repo/__init__.py

@@ -485,7 +485,7 @@ def _collect_graph(self, stages):
                     continue
 
                 dep_key = dep.path_info.parts
-                overlapping = list(n.value for n in outs.prefixes(dep_key))
+                overlapping = [n.value for n in outs.prefixes(dep_key)]
                 if outs.has_subtrie(dep_key):
                     overlapping.extend(outs.values(prefix=dep_key))
 

dvc/repo/plots/data.py

@@ -196,7 +196,7 @@ def raw(self, header=True, **kwargs):
             )
 
         fieldnames = reader.fieldnames
-        data = [row for row in reader]
+        data = list(reader)
 
         return [
             OrderedDict([(field, data_point[field]) for field in fieldnames])

dvc/repo/tree.py

@@ -142,7 +142,7 @@ def _walk(self, root, trie, topdown=True):
         files = []
 
         root_len = len(root.parts)
-        for key, out in trie.iteritems(prefix=root.parts):
+        for key, out in trie.iteritems(prefix=root.parts):  # noqa: B301
             if key == root.parts:
                 continue
 
@@ -153,14 +153,12 @@ def _walk(self, root, trie, topdown=True):
 
             files.append(name)
 
-        if topdown:
-            dirs = list(dirs)
-            yield root.fspath, dirs, files
+        assert topdown
+        dirs = list(dirs)
+        yield root.fspath, dirs, files
 
-            for dname in dirs:
-                yield from self._walk(root / dname, trie)
-        else:
-            assert False
+        for dname in dirs:
+            yield from self._walk(root / dname, trie)
 
     def walk(
         self, top, topdown=True, onerror=None, download_callback=None, **kwargs

dvc/scm/git/__init__.py

@@ -37,24 +37,22 @@ def update_git(self, op_code, cur_count, max_count=None, message=""):
     def code2desc(op_code):
         from git import RootUpdateProgress as OP
 
-        ops = dict(
-            (
-                (OP.COUNTING, "Counting"),
-                (OP.COMPRESSING, "Compressing"),
-                (OP.WRITING, "Writing"),
-                (OP.RECEIVING, "Receiving"),
-                (OP.RESOLVING, "Resolving"),
-                (OP.FINDING_SOURCES, "Finding sources"),
-                (OP.CHECKING_OUT, "Checking out"),
-                (OP.CLONE, "Cloning"),
-                (OP.FETCH, "Fetching"),
-                (OP.UPDWKTREE, "Updating working tree"),
-                (OP.REMOVE, "Removing"),
-                (OP.PATHCHANGE, "Changing path"),
-                (OP.URLCHANGE, "Changing URL"),
-                (OP.BRANCHCHANGE, "Changing branch"),
-            )
-        )
+        ops = {
+            OP.COUNTING: "Counting",
+            OP.COMPRESSING: "Compressing",
+            OP.WRITING: "Writing",
+            OP.RECEIVING: "Receiving",
+            OP.RESOLVING: "Resolving",
+            OP.FINDING_SOURCES: "Finding sources",
+            OP.CHECKING_OUT: "Checking out",
+            OP.CLONE: "Cloning",
+            OP.FETCH: "Fetching",
+            OP.UPDWKTREE: "Updating working tree",
+            OP.REMOVE: "Removing",
+            OP.PATHCHANGE: "Changing path",
+            OP.URLCHANGE: "Changing URL",
+            OP.BRANCHCHANGE: "Changing branch",
+        }
         return ops.get(op_code & OP.OP_MASK, "")
 
 

dvc/scm/tree.py

@@ -38,7 +38,8 @@ def walk_files(self, top):
 class WorkingTree(BaseTree):
     """Proxies the repo file access methods to working tree files"""
 
-    def __init__(self, repo_root=os.getcwd()):
+    def __init__(self, repo_root=None):
+        repo_root = repo_root or os.getcwd()
         self.repo_root = repo_root
 
     @property

dvc/stage/serialize.py

@@ -111,7 +111,7 @@ def _serialize_params_values(params: List[ParamsDependency]):
 def to_pipeline_file(stage: "PipelineStage"):
     wdir = resolve_wdir(stage.wdir, stage.path)
     params, deps = split_params_deps(stage)
-    deps = sorted([d.def_path for d in deps])
+    deps = sorted(d.def_path for d in deps)
     params = _serialize_params_keys(params)
 
     outs, metrics, plots = _serialize_outs(stage.outs)

setup.cfg

@@ -6,8 +6,13 @@ ignore=
     E203, # Whitespace before ':'
     E266, # Too many leading '#' for block comment
     W503, # Line break occurred before a binary operator
+    P1,  # unindexed parameters in the str.format, see:
+    # https://pypi.org/project/flake8-string-format/
 max_line_length=79
-select=B,C,E,F,W,T4,B9
+max-complexity = 15
+select=B,C,E,F,W,T4,B902,T,P
+show_source=true
+count=true
 
 [isort]
 include_trailing_comma=true

setup.py

@@ -123,6 +123,9 @@ def run(self):
     "moto==1.3.14.dev464",
     "rangehttpserver==1.2.0",
     "beautifulsoup4==4.4.0",
+    "flake8-bugbear",
+    "flake8-comprehensions",
+    "flake8-string-format",
 ]
 
 if (sys.version_info) >= (3, 6):

tests/func/test_dvcfile.py

@@ -157,7 +157,7 @@ def test_stage_collection(tmp_dir, dvc):
         always_changed=True,
         single_stage=True,
     )
-    assert {s for s in dvc.stages} == {stage1, stage3, stage2}
+    assert set(dvc.stages) == {stage1, stage3, stage2}
 
 
 def test_remove_stage(tmp_dir, dvc, run_copy):

tests/func/test_gc.py

@@ -52,7 +52,7 @@ def _test_gc(self):
 class TestGCBranchesTags(TestDvcGit):
     def _check_cache(self, num):
         total = 0
-        for root, dirs, files in os.walk(os.path.join(".dvc", "cache")):
+        for _, _, files in os.walk(os.path.join(".dvc", "cache")):
             total += len(files)
         self.assertEqual(total, num)
 
@@ -112,7 +112,7 @@ def test(self):
 class TestGCMultipleDvcRepos(TestDvcGit):
     def _check_cache(self, num):
         total = 0
-        for root, dirs, files in os.walk(os.path.join(".dvc", "cache")):
+        for _, _, files in os.walk(os.path.join(".dvc", "cache")):
             total += len(files)
         self.assertEqual(total, num)