From 3ca6cc3d57e514cd99dd4bc60b48a92d56e2ab9b Mon Sep 17 00:00:00 2001
From: Fokko Driesprong <fokko@apache.org>
Date: Thu, 21 May 2020 19:25:39 +0200
Subject: [PATCH] Python: Enable mypy type validation (#1041)

---
 .gitignore                             |  3 +++
 python/iceberg/api/types/type_util.py  | 10 +++++-----
 python/iceberg/core/base_table_scan.py |  4 ++--
 python/iceberg/core/manifest_reader.py |  4 ++--
 python/tox.ini                         | 13 ++++++++++---
 5 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/.gitignore b/.gitignore
index 13767d622f82..f1e459c8cb0a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -38,3 +38,6 @@ bin/
 
 # Hive/metastore files
 metastore_db/
+
+# Python stuff
+python/.mypy_cache/
diff --git a/python/iceberg/api/types/type_util.py b/python/iceberg/api/types/type_util.py
index ca52a7db9c6f..68f0f595acf3 100644
--- a/python/iceberg/api/types/type_util.py
+++ b/python/iceberg/api/types/type_util.py
@@ -16,6 +16,7 @@
 # under the License.
 
 import math
+from typing import List
 
 from .type import (Type,
                    TypeID)
@@ -238,7 +239,6 @@ def get(self):
         return self.visitor.field(self.field, VisitFuture(self.field.type, self.visitor).get)
 
 
-@staticmethod
 def decimal_required_bytes(precision):
     if precision < 0 or precision > 40:
         raise RuntimeError("Unsupported decimal precision: %s" % precision)
@@ -451,11 +451,11 @@ def write_compatibility_errors(read_schema, write_schema):
     def read_compatibility_errors(read_schema, write_schema):
         visit(write_schema, CheckCompatibility(read_schema, False))
 
-    NO_ERRORS = []
+    NO_ERRORS: List[str] = []
 
     def __init__(self, schema, check_ordering):
         self.schema = schema
-        self.check_ordering
+        self.check_ordering = check_ordering
         self.current_type = None
 
     def schema(self, schema, struct_result):
@@ -498,10 +498,10 @@ def struct(self, struct, field_results):
 
         return errors
 
-    def field(self, field, field_result):
+    def field(self, field, field_result) -> List[str]:
         struct = self.current_type.as_struct_type()
         curr_field = struct.field(field.field_id)
-        errors = list()
+        errors = []
 
         if curr_field is None:
             if not field.is_optional:
diff --git a/python/iceberg/core/base_table_scan.py b/python/iceberg/core/base_table_scan.py
index d205364aad15..e64038d3136a 100644
--- a/python/iceberg/core/base_table_scan.py
+++ b/python/iceberg/core/base_table_scan.py
@@ -34,9 +34,9 @@
 
 class BaseTableScan(CloseableGroup, TableScan):
     DATE_FORMAT = "%Y-%m-%d %H:%M:%S.%f"
-    SNAPSHOT_COLUMNS = ["snapshot_id", "file_path", "file_ordinal", "file_format", "block_size_in_bytes",
+    SNAPSHOT_COLUMNS = ("snapshot_id", "file_path", "file_ordinal", "file_format", "block_size_in_bytes",
                         "file_size_in_bytes", "record_count", "partition", "value_counts", "null_value_counts",
-                        "lower_bounds", "upper_bounds"]
+                        "lower_bounds", "upper_bounds")
 
     def new_refined_scan(self, ops, table, schema, snapshot_id, row_filter,
                          case_sensitive, selected_columns, options, minused_cols):
diff --git a/python/iceberg/core/manifest_reader.py b/python/iceberg/core/manifest_reader.py
index 6a4581293a6f..e648684f29fe 100644
--- a/python/iceberg/core/manifest_reader.py
+++ b/python/iceberg/core/manifest_reader.py
@@ -33,8 +33,8 @@
 
 
 class ManifestReader(CloseableGroup, Filterable):
-    ALL_COLUMNS = ["*"]
-    CHANGE_COLUMNS = ["file_path", "file_format", "partition", "record_count", "file_size_in_bytes"]
+    ALL_COLUMNS = ("*",)
+    CHANGE_COLUMNS = ("file_path", "file_format", "partition", "record_count", "file_size_in_bytes")
 
     @staticmethod
     def read(file, spec_lookup=None):
diff --git a/python/tox.ini b/python/tox.ini
index 9c49b0bd33cc..b011511130b2 100644
--- a/python/tox.ini
+++ b/python/tox.ini
@@ -39,9 +39,11 @@ deps =
     .
     {[testenv:flake8]deps}
     {[testenv:bandit]deps}
+    {[testenv:mypy]deps}
 commands =
     {[testenv:flake8]commands}
     {[testenv:bandit]commands}
+    {[testenv:mypy]commands}
 
 [testenv:flake8]
 basepython = python3
@@ -53,6 +55,14 @@ deps =
 commands =
     flake8 iceberg setup.py tests
 
+[testenv:mypy]
+basepython = python3
+skip_install = true
+deps =
+    mypy
+commands =
+    mypy --ignore-missing-imports iceberg/
+
 [testenv:bandit]
 basepython = python3
 skip_install = true
@@ -77,9 +87,6 @@ commands =
 # commands =
 #     python -m http.server {posargs}
 
-[bandit]
-skips = B104
-
 [flake8]
 ignore = E501,W503
 exclude =