Bump mypy, ruff and black (zenml-io#1963)

* bump mypy and ruff

* bump black

* mypy narrowed ignores

* formatting

* mypy fixes

* mypy fixes

* mypy fixes (final)

pyproject.toml

@@ -75,8 +75,8 @@ ipinfo = { version = ">=4.4.3", optional = true}
 # Optional dependencies for project templates
 copier = { version = ">=8.1.0", optional = true }
 jinja2-time = { version = "^0.2.0", optional = true }
-black = { version = "^23.3.0", optional = true }
-ruff = { version = "^0.0.285", optional = true }
+black = { version = "^23.10.0", optional = true }
+ruff = { version = "^0.1.0", optional = true }
 
 # Optional terraform dependency for stack recipes and ZenServer deployments
 python-terraform = { version = "^0.10.1", optional = true }
@@ -123,7 +123,7 @@ adlfs = { version = ">=2021.10.0", optional = true }
 # Optional development dependencies
 bandit = { version = "^1.7.5", optional = true }
 coverage = { extras = ["toml"], version = "^5.5", optional = true }
-mypy = { version = "1.5.1", optional = true }
+mypy = { version = "1.6.1", optional = true }
 pre-commit = { version = "^2.14.0", optional = true }
 pyment = { version = "^0.3.3", optional = true }
 tox = { version = "^3.24.3", optional = true }

src/zenml/integrations/aws/service_connectors/aws_service_connector.py

@@ -31,7 +31,7 @@
 from typing import Any, Dict, List, Optional, Tuple, cast
 
 import boto3
-from aws_profile_manager import Common  # type: ignore[import]
+from aws_profile_manager import Common  # type: ignore[import-untyped]
 from botocore.client import BaseClient
 from botocore.exceptions import BotoCoreError, ClientError
 from botocore.signers import RequestSigner

src/zenml/integrations/evidently/__init__.py

@@ -34,7 +34,7 @@
 
 # Fix numba errors in Docker and suppress logs and deprecation warning spam
 try:
-    from numba.core.errors import (  # type: ignore[import]
+    from numba.core.errors import (  # type: ignore[import-untyped]
         NumbaDeprecationWarning,
         NumbaPendingDeprecationWarning,
     )

src/zenml/integrations/evidently/column_mapping.py

@@ -15,7 +15,7 @@
 
 from typing import List, Optional, Sequence, Union
 
-from evidently import ColumnMapping  # type: ignore[import]
+from evidently import ColumnMapping  # type: ignore[import-untyped]
 from pydantic import BaseModel
 
 

src/zenml/integrations/evidently/data_validators/evidently_data_validator.py

@@ -125,8 +125,10 @@ def _download_nltk_data() -> None:
             ImportError: if NLTK is not installed.
         """
         try:
-            import nltk  # type: ignore[import]
-            from nltk.data import path as nltk_path  # type: ignore[import]
+            import nltk  # type: ignore[import-untyped]
+            from nltk.data import (  # type: ignore[import-untyped]
+                path as nltk_path,
+            )
         except ImportError:
             raise ImportError(
                 "NLTK is not installed. Please install NLTK to use "

src/zenml/integrations/evidently/metrics.py

@@ -22,15 +22,17 @@
     Union,
 )
 
-from evidently import metric_preset, metrics  # type: ignore[import]
-from evidently.metric_preset.metric_preset import (  # type: ignore[import]
+from evidently import metric_preset, metrics  # type: ignore[import-untyped]
+from evidently.metric_preset.metric_preset import (  # type: ignore[import-untyped]
     MetricPreset,
 )
-from evidently.metrics.base_metric import (  # type: ignore[import]
+from evidently.metrics.base_metric import (  # type: ignore[import-untyped]
     Metric,
     generate_column_metrics,
 )
-from evidently.utils.generators import BaseGenerator  # type: ignore[import]
+from evidently.utils.generators import (  # type: ignore[import-untyped]
+    BaseGenerator,
+)
 from pydantic import BaseModel, Field
 
 from zenml.logger import get_logger

src/zenml/integrations/evidently/tests.py

@@ -22,15 +22,17 @@
     Union,
 )
 
-from evidently import test_preset, tests  # type: ignore[import]
-from evidently.test_preset.test_preset import (  # type: ignore[import]
+from evidently import test_preset, tests  # type: ignore[import-untyped]
+from evidently.test_preset.test_preset import (  # type: ignore[import-untyped]
     TestPreset,
 )
-from evidently.tests.base_test import (  # type: ignore[import]
+from evidently.tests.base_test import (  # type: ignore[import-untyped]
     Test,
     generate_column_tests,
 )
-from evidently.utils.generators import BaseGenerator  # type: ignore[import]
+from evidently.utils.generators import (  # type: ignore[import-untyped]
+    BaseGenerator,
+)
 from pydantic import BaseModel, Field
 
 from zenml.logger import get_logger

src/zenml/integrations/feast/feature_stores/feast_feature_store.py

@@ -17,8 +17,8 @@
 
 import pandas as pd
 import redis
-from feast import FeatureStore  # type: ignore[import]
-from feast.infra.registry.registry import Registry  # type: ignore[import]
+from feast import FeatureStore  # type: ignore
+from feast.infra.registry.base_registry import BaseRegistry  # type: ignore
 
 from zenml.feature_stores.base_feature_store import BaseFeatureStore
 from zenml.integrations.feast.flavors.feast_feature_store_flavor import (
@@ -177,7 +177,7 @@ def get_project(self) -> str:
         fs = FeatureStore(repo_path=self.config.feast_repo)
         return str(fs.project)
 
-    def get_registry(self) -> Registry:
+    def get_registry(self) -> BaseRegistry:
         """Returns the feature store registry.
 
         Raise:

src/zenml/integrations/great_expectations/data_validators/ge_data_validator.py

@@ -18,21 +18,23 @@
 
 import pandas as pd
 import yaml
-from great_expectations.checkpoint.types.checkpoint_result import (  # type: ignore[import]
+from great_expectations.checkpoint.types.checkpoint_result import (  # type: ignore[import-untyped]
     CheckpointResult,
 )
-from great_expectations.core import ExpectationSuite  # type: ignore[import]
-from great_expectations.data_context.data_context import (  # type: ignore[import]
+from great_expectations.core import (  # type: ignore[import-untyped]
+    ExpectationSuite,
+)
+from great_expectations.data_context.data_context import (  # type: ignore[import-untyped]
     BaseDataContext,
     DataContext,
 )
-from great_expectations.data_context.types.base import (  # type: ignore[import]
+from great_expectations.data_context.types.base import (  # type: ignore[import-untyped]
     DataContextConfig,
 )
-from great_expectations.data_context.types.resource_identifiers import (  # type: ignore[import]
+from great_expectations.data_context.types.resource_identifiers import (  # type: ignore[import-untyped]
     ExpectationSuiteIdentifier,
 )
-from great_expectations.profile.user_configurable_profiler import (  # type: ignore[import]
+from great_expectations.profile.user_configurable_profiler import (  # type: ignore[import-untyped]
     UserConfigurableProfiler,
 )
 

src/zenml/integrations/great_expectations/ge_store_backend.py

@@ -17,11 +17,11 @@
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple, cast
 
-from great_expectations.data_context.store.tuple_store_backend import (  # type: ignore[import]
+from great_expectations.data_context.store.tuple_store_backend import (  # type: ignore[import-untyped]
     TupleStoreBackend,
     filter_properties_dict,
 )
-from great_expectations.exceptions import (  # type: ignore[import]
+from great_expectations.exceptions import (  # type: ignore[import-untyped]
     InvalidKeyError,
     StoreBackendError,
 )

src/zenml/integrations/great_expectations/materializers/ge_materializer.py

@@ -16,21 +16,23 @@
 import os
 from typing import TYPE_CHECKING, Any, ClassVar, Dict, Tuple, Type, Union, cast
 
-from great_expectations.checkpoint.types.checkpoint_result import (  # type: ignore[import]
+from great_expectations.checkpoint.types.checkpoint_result import (  # type: ignore[import-untyped]
     CheckpointResult,
 )
-from great_expectations.core import ExpectationSuite  # type: ignore[import]
-from great_expectations.core.expectation_validation_result import (  # type: ignore[import]
+from great_expectations.core import (  # type: ignore[import-untyped]
+    ExpectationSuite,
+)
+from great_expectations.core.expectation_validation_result import (  # type: ignore[import-untyped]
     ExpectationSuiteValidationResult,
 )
-from great_expectations.data_context.types.base import (  # type: ignore[import]
+from great_expectations.data_context.types.base import (  # type: ignore[import-untyped]
     CheckpointConfig,
 )
-from great_expectations.data_context.types.resource_identifiers import (  # type: ignore[import]  # type: ignore[import]
+from great_expectations.data_context.types.resource_identifiers import (  # type: ignore[import-untyped]
     ExpectationSuiteIdentifier,
     ValidationResultIdentifier,
 )
-from great_expectations.types import (  # type: ignore[import]
+from great_expectations.types import (  # type: ignore[import-untyped]
     SerializableDictDot,
 )
 

src/zenml/integrations/great_expectations/steps/ge_profiler.py

@@ -16,7 +16,9 @@
 from typing import Any, Dict, Optional
 
 import pandas as pd
-from great_expectations.core import ExpectationSuite  # type: ignore[import]
+from great_expectations.core import (  # type: ignore[import-untyped]
+    ExpectationSuite,
+)
 
 from zenml import step
 from zenml.integrations.great_expectations.data_validators.ge_data_validator import (

src/zenml/integrations/great_expectations/steps/ge_validator.py

@@ -16,7 +16,7 @@
 from typing import Any, Dict, List, Optional
 
 import pandas as pd
-from great_expectations.checkpoint.types.checkpoint_result import (  # type: ignore[import]
+from great_expectations.checkpoint.types.checkpoint_result import (  # type: ignore[import-untyped]
     CheckpointResult,
 )
 

src/zenml/integrations/great_expectations/utils.py

@@ -16,10 +16,10 @@
 from typing import Optional
 
 import pandas as pd
-from great_expectations.core.batch import (  # type: ignore[import]
+from great_expectations.core.batch import (  # type: ignore[import-untyped]
     RuntimeBatchRequest,
 )
-from great_expectations.data_context.data_context import (  # type: ignore[import]
+from great_expectations.data_context.data_context import (  # type: ignore[import-untyped]
     BaseDataContext,
 )
 

src/zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py

@@ -18,7 +18,10 @@
 from tempfile import TemporaryDirectory
 from typing import Any, ClassVar, Dict, Tuple, Type
 
-from transformers import AutoConfig, PreTrainedModel  # type: ignore [import]
+from transformers import (  # type: ignore [import-untyped]
+    AutoConfig,
+    PreTrainedModel,
+)
 
 from zenml.enums import ArtifactType
 from zenml.materializers.base_materializer import BaseMaterializer

src/zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py

@@ -18,7 +18,10 @@
 from tempfile import TemporaryDirectory
 from typing import Any, ClassVar, Dict, Tuple, Type
 
-from transformers import AutoConfig, TFPreTrainedModel  # type: ignore [import]
+from transformers import (  # type: ignore [import-untyped]
+    AutoConfig,
+    TFPreTrainedModel,
+)
 
 from zenml.enums import ArtifactType
 from zenml.materializers.base_materializer import BaseMaterializer

src/zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py

@@ -17,8 +17,8 @@
 from tempfile import TemporaryDirectory
 from typing import Any, ClassVar, Tuple, Type
 
-from transformers import AutoTokenizer  # type: ignore [import]
-from transformers.tokenization_utils_base import (  # type: ignore [import]
+from transformers import AutoTokenizer  # type: ignore [import-untyped]
+from transformers.tokenization_utils_base import (  # type: ignore [import-untyped]
     PreTrainedTokenizerBase,
 )
 

src/zenml/integrations/label_studio/annotators/label_studio_annotator.py

@@ -19,7 +19,7 @@
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple, cast
 
-from label_studio_sdk import Client, Project  # type: ignore[import]
+from label_studio_sdk import Client, Project  # type: ignore[import-not-found]
 
 from zenml.annotators.base_annotator import BaseAnnotator
 from zenml.artifact_stores.base_artifact_store import BaseArtifactStore

src/zenml/integrations/neptune/experiment_trackers/run_state.py

@@ -16,7 +16,7 @@
 from hashlib import md5
 from typing import TYPE_CHECKING, Any, List, Optional
 
-import neptune  # type: ignore[import]
+import neptune  # type: ignore[import-untyped]
 
 import zenml
 from zenml.client import Client

src/zenml/integrations/tensorboard/services/tensorboard_service.py

@@ -15,8 +15,10 @@
 
 from typing import Any, Dict, Union
 
-from tensorboard import default, program  # type: ignore [import]
-from tensorboard.uploader import uploader_subcommand  # type: ignore [import]
+from tensorboard import default, program  # type: ignore [import-untyped]
+from tensorboard.uploader import (  # type: ignore [import-untyped]
+    uploader_subcommand,
+)
 
 from zenml.logger import get_logger
 from zenml.services import (

src/zenml/integrations/tensorboard/visualizers/tensorboard_visualizer.py

@@ -19,8 +19,8 @@
 
 import psutil
 from rich import print
-from tensorboard import notebook  # type: ignore [import]
-from tensorboard.manager import (  # type: ignore [import]
+from tensorboard import notebook  # type: ignore [import-untyped]
+from tensorboard.manager import (  # type: ignore [import-untyped]
     TensorBoardInfo,
     get_all,
 )

src/zenml/integrations/tensorflow/__init__.py

@@ -35,7 +35,7 @@ def activate(cls) -> None:
             not platform.system() == "Darwin"
             or not platform.machine() == "arm64"
         ):
-            import tensorflow_io  # type: ignore [import]
+            import tensorflow_io  # type: ignore [import-untyped]
 
         from zenml.integrations.tensorflow import materializers  # noqa
 

src/zenml/integrations/vault/secrets_manager/vault_secrets_manager.py

@@ -14,8 +14,8 @@
 """Implementation of the HashiCorp Vault Secrets Manager integration."""
 from typing import Any, ClassVar, List, Set, cast
 
-import hvac  # type: ignore[import]
-from hvac.exceptions import InvalidPath  # type: ignore[import]
+import hvac  # type: ignore[import-untyped]
+from hvac.exceptions import InvalidPath  # type: ignore[import-untyped]
 
 from zenml.constants import ZENML_SCHEMA_NAME
 from zenml.exceptions import SecretExistsError

src/zenml/utils/secret_utils.py

@@ -115,7 +115,7 @@ def is_secret_field(field: "ModelField") -> bool:
     Returns:
         `True` if the field contains sensitive information, `False` otherwise.
     """
-    return field.field_info.extra.get(PYDANTIC_SENSITIVE_FIELD_MARKER, False)
+    return field.field_info.extra.get(PYDANTIC_SENSITIVE_FIELD_MARKER, False)  # type: ignore[no-any-return]
 
 
 def is_clear_text_field(field: "ModelField") -> bool:
@@ -127,4 +127,4 @@ def is_clear_text_field(field: "ModelField") -> bool:
     Returns:
         `True` if the field prevents secret references, `False` otherwise.
     """
-    return field.field_info.extra.get(PYDANTIC_CLEAR_TEXT_FIELD_MARKER, False)
+    return field.field_info.extra.get(PYDANTIC_CLEAR_TEXT_FIELD_MARKER, False)  # type: ignore[no-any-return]

src/zenml/zen_server/utils.py

@@ -274,7 +274,7 @@ def get_ip_location(ip_address: str) -> Tuple[str, str, str]:
     Returns:
         A tuple of city, region, country.
     """
-    import ipinfo  # type: ignore[import]
+    import ipinfo  # type: ignore[import-untyped]
 
     try:
         handler = ipinfo.getHandler()

src/zenml/zen_stores/secrets_stores/hashicorp_secrets_store.py

@@ -28,8 +28,11 @@
 )
 from uuid import UUID
 
-import hvac  # type: ignore[import]
-from hvac.exceptions import InvalidPath, VaultError  # type: ignore[import]
+import hvac  # type: ignore[import-untyped]
+from hvac.exceptions import (  # type: ignore[import-untyped]
+    InvalidPath,
+    VaultError,
+)
 from pydantic import SecretStr
 
 from zenml.analytics.enums import AnalyticsEvent

tests/harness/deployment/server_docker_compose.py

@@ -157,7 +157,7 @@ def up(self) -> None:
         Raises:
             RuntimeError: If the deployment could not be started.
         """
-        from compose.cli.main import (  # type: ignore[import]
+        from compose.cli.main import (  # type: ignore[import-not-found]
             TopLevelCommand,
             project_from_options,
         )