Chore: replace autoflake+black+isort with ruff (TobikoData#2584)

* Chore: replace autoflake+black+isort with ruff

* Fixup

* Fixup

* Explicitly re-export names in init files

* Cleanup

* Rename PyDataFrame to PySparkDataFrame

.pre-commit-config.yaml

@@ -1,34 +1,20 @@
 repos:
   - repo: local
     hooks:
-      - id: autoflake
-        name: autoflake
-        entry: autoflake
-        language: system
-        "types": [python]
+      - id: ruff
+        name: ruff
+        entry: ruff check --force-exclude --fix --ignore E721 --ignore E741
+        language: python
+        types_or: [python, pyi]
         require_serial: true
         files: &files ^(sqlmesh/|tests/|web/|examples/|setup.py)
-      - id: isort
-        name: isort
-        entry: isort
-        language: system
-        "types": [python]
-        files: *files
-        require_serial: true
-      - id: black
-        name: black
-        language: system
-        args:
-          - --target-version
-          - py37
-          - --line-length
-          - "100"
-        entry: black
+      - id: ruff-format
+        name: ruff-format
+        entry: ruff format --force-exclude --line-length 100
+        language: python
+        types_or: [python, pyi]
         require_serial: true
         files: *files
-        types_or:
-          - python
-          - pyi
       - id: mypy
         name: mypy
         language: system

Makefile

@@ -22,7 +22,7 @@ py-style:
 	SKIP=prettier,eslint pre-commit run --all-files
 
 ui-style:
-	SKIP=autoflake,isort,black,mypy pre-commit run --all-files
+	SKIP=ruff,ruff-format,mypy pre-commit run --all-files
 
 doc-test:
 	PYTEST_PLUGINS=tests.common_fixtures pytest --doctest-modules sqlmesh/core sqlmesh/utils

examples/sushi/config.py

@@ -127,7 +127,7 @@
 
 CATALOGS = {
     "in_memory": ":memory:",
-    "other_catalog": f":memory:",
+    "other_catalog": ":memory:",
 }
 
 local_catalogs = Config(

examples/sushi/models/raw_marketing.py

@@ -4,7 +4,6 @@
 
 import numpy as np
 import pandas as pd
-from helper import iter_dates  # type: ignore
 from sqlglot import exp
 
 from sqlmesh import ExecutionContext, model

setup.cfg

@@ -98,16 +98,3 @@ ignore_missing_imports = True
 
 [mypy-pydantic_core.*]
 ignore_missing_imports = True
-
-[autoflake]
-in-place = True
-expand-star-imports = True
-remove-all-unused-imports = True
-ignore-init-module-imports = True
-remove-duplicate-keys = True
-remove-unused-variables = True
-quiet = True
-
-[isort]
-profile=black
-known_first_party=sqlmesh

setup.py

@@ -59,19 +59,17 @@
         ],
         "dev": [
             f"apache-airflow=={os.environ.get('AIRFLOW_VERSION', '2.9.1')}",
-            "autoflake==1.7.7",
             "agate==1.7.1",
             "beautifulsoup4",
-            "black==24.4.2",
+            "ruff~=0.4.0",
             "cryptography~=42.0.4",
             "dbt-core",
             "dbt-duckdb>=1.7.1",
             "Faker",
             "google-auth",
             "google-cloud-bigquery",
             "google-cloud-bigquery-storage",
-            "isort==5.10.1",
-            "mypy~=1.8.0",
+            "mypy~=1.10.0",
             "pre-commit",
             "pandas-stubs",
             "psycopg2-binary",

sqlmesh/__init__.py

@@ -1,3 +1,4 @@
+# ruff: noqa: E402
 """
 .. include:: ../README.md
 """
@@ -18,16 +19,19 @@
 extend_sqlglot()
 
 from sqlmesh.core import constants as c
-from sqlmesh.core.config import Config
-from sqlmesh.core.context import Context, ExecutionContext
-from sqlmesh.core.engine_adapter import EngineAdapter
-from sqlmesh.core.macros import SQL, macro
-from sqlmesh.core.model import Model, model
-from sqlmesh.core.snapshot import Snapshot
-from sqlmesh.utils import debug_mode_enabled, enable_debug_mode
+from sqlmesh.core.config import Config as Config
+from sqlmesh.core.context import Context as Context, ExecutionContext as ExecutionContext
+from sqlmesh.core.engine_adapter import EngineAdapter as EngineAdapter
+from sqlmesh.core.macros import SQL as SQL, macro as macro
+from sqlmesh.core.model import Model as Model, model as model
+from sqlmesh.core.snapshot import Snapshot as Snapshot
+from sqlmesh.utils import (
+    debug_mode_enabled as debug_mode_enabled,
+    enable_debug_mode as enable_debug_mode,
+)
 
 try:
-    from sqlmesh._version import __version__, __version_tuple__  # type: ignore
+    from sqlmesh._version import __version__ as __version__, __version_tuple__ as __version_tuple__
 except ImportError:
     pass
 

sqlmesh/cli/__init__.py

@@ -17,7 +17,7 @@
 
 
 def error_handler(
-    func: t.Callable[..., DECORATOR_RETURN_TYPE]
+    func: t.Callable[..., DECORATOR_RETURN_TYPE],
 ) -> t.Callable[..., DECORATOR_RETURN_TYPE]:
     @wraps(func)
     def wrapper(*args: t.List[t.Any], **kwargs: t.Any) -> DECORATOR_RETURN_TYPE:

sqlmesh/core/audit/__init__.py

@@ -4,12 +4,12 @@
 
 from sqlmesh.core.audit import builtin
 from sqlmesh.core.audit.definition import (
-    Audit,
-    AuditResult,
-    ModelAudit,
-    StandaloneAudit,
-    load_audit,
-    load_multiple_audits,
+    Audit as Audit,
+    AuditResult as AuditResult,
+    ModelAudit as ModelAudit,
+    StandaloneAudit as StandaloneAudit,
+    load_audit as load_audit,
+    load_multiple_audits as load_multiple_audits,
 )
 
 

sqlmesh/core/config/__init__.py

@@ -1,34 +1,37 @@
-from sqlmesh.core.config.categorizer import AutoCategorizationMode, CategorizerConfig
-from sqlmesh.core.config.common import EnvironmentSuffixTarget
+from sqlmesh.core.config.categorizer import (
+    AutoCategorizationMode as AutoCategorizationMode,
+    CategorizerConfig as CategorizerConfig,
+)
+from sqlmesh.core.config.common import EnvironmentSuffixTarget as EnvironmentSuffixTarget
 from sqlmesh.core.config.connection import (
-    BigQueryConnectionConfig,
-    ConnectionConfig,
-    DatabricksConnectionConfig,
-    DuckDBConnectionConfig,
-    GCPPostgresConnectionConfig,
-    MotherDuckConnectionConfig,
-    MSSQLConnectionConfig,
-    MySQLConnectionConfig,
-    PostgresConnectionConfig,
-    RedshiftConnectionConfig,
-    SnowflakeConnectionConfig,
-    SparkConnectionConfig,
-    parse_connection_config,
+    BigQueryConnectionConfig as BigQueryConnectionConfig,
+    ConnectionConfig as ConnectionConfig,
+    DatabricksConnectionConfig as DatabricksConnectionConfig,
+    DuckDBConnectionConfig as DuckDBConnectionConfig,
+    GCPPostgresConnectionConfig as GCPPostgresConnectionConfig,
+    MotherDuckConnectionConfig as MotherDuckConnectionConfig,
+    MSSQLConnectionConfig as MSSQLConnectionConfig,
+    MySQLConnectionConfig as MySQLConnectionConfig,
+    PostgresConnectionConfig as PostgresConnectionConfig,
+    RedshiftConnectionConfig as RedshiftConnectionConfig,
+    SnowflakeConnectionConfig as SnowflakeConnectionConfig,
+    SparkConnectionConfig as SparkConnectionConfig,
+    parse_connection_config as parse_connection_config,
 )
-from sqlmesh.core.config.gateway import GatewayConfig
+from sqlmesh.core.config.gateway import GatewayConfig as GatewayConfig
 from sqlmesh.core.config.loader import (
-    load_config_from_paths,
-    load_config_from_yaml,
-    load_configs,
+    load_config_from_paths as load_config_from_paths,
+    load_config_from_yaml as load_config_from_yaml,
+    load_configs as load_configs,
 )
-from sqlmesh.core.config.migration import MigrationConfig
-from sqlmesh.core.config.model import ModelDefaultsConfig
-from sqlmesh.core.config.plan import PlanConfig
-from sqlmesh.core.config.root import Config
-from sqlmesh.core.config.run import RunConfig
+from sqlmesh.core.config.migration import MigrationConfig as MigrationConfig
+from sqlmesh.core.config.model import ModelDefaultsConfig as ModelDefaultsConfig
+from sqlmesh.core.config.plan import PlanConfig as PlanConfig
+from sqlmesh.core.config.root import Config as Config
+from sqlmesh.core.config.run import RunConfig as RunConfig
 from sqlmesh.core.config.scheduler import (
-    AirflowSchedulerConfig,
-    BuiltInSchedulerConfig,
-    CloudComposerSchedulerConfig,
-    MWAASchedulerConfig,
+    AirflowSchedulerConfig as AirflowSchedulerConfig,
+    BuiltInSchedulerConfig as BuiltInSchedulerConfig,
+    CloudComposerSchedulerConfig as CloudComposerSchedulerConfig,
+    MWAASchedulerConfig as MWAASchedulerConfig,
 )

sqlmesh/core/config/loader.py

@@ -118,7 +118,7 @@ def load_config_from_paths(
 
     supported_model_defaults = ModelDefaultsConfig.all_fields()
     for default in non_python_config_dict.get("model_defaults", {}):
-        if not default in supported_model_defaults:
+        if default not in supported_model_defaults:
             raise ConfigError(
                 f"'{default}' is not a valid model default configuration key. Please remove it from the `model_defaults` specification in your config file."
             )