Fix isort in our code and automated linter (#1227)

.github/workflows/lint-python.yml

@@ -57,4 +57,5 @@ jobs:
       - uses: actions/checkout@v2
       - uses: isort/isort-action@master
         with:
-            configuration: "-l 100 --profile black"
+          isort-version: "5.12.0"
+          configuration: "-l 100 --profile black --check-only --diff"

examples/run_notebook.py

@@ -7,9 +7,10 @@
 from pathlib import Path
 from typing import List
 
-from aqueduct import Client, get_apikey
 from aqueduct.error import InvalidUserArgumentException
 
+from aqueduct import Client, get_apikey
+
 """
 See README.md for details about this script.
 """

integration_tests/backend/setup/flow_with_metrics_and_checks.py

@@ -3,9 +3,10 @@
 # with a check to enforce the row-count is larger than 0.
 # This workflow is published twice.
 ###
-import aqueduct
 from aqueduct.constants.enums import CheckSeverity
 
+import aqueduct
+
 
 def setup_flow_with_metrics_and_checks(
     client: aqueduct.Client,

integration_tests/backend/utils.py

@@ -1,9 +1,10 @@
 import time
 from typing import Optional
 
-import aqueduct
 from aqueduct.constants.enums import ExecutionStatus
 
+import aqueduct
+
 
 def wait_for_flow_runs(
     client: aqueduct.Client, flow_id: str, num_runs: int = 1, expect_success: Optional[bool] = None

integration_tests/no_concurrency/sdk/conftest.py

@@ -1,10 +1,10 @@
 import os
 
 import pytest
+from aqueduct.models.dag import DAG, Metadata
 
 import aqueduct
 from aqueduct import globals
-from aqueduct.models.dag import DAG, Metadata
 
 API_KEY_ENV_NAME = "API_KEY"
 SERVER_ADDR_ENV_NAME = "SERVER_ADDRESS"

integration_tests/no_concurrency/sdk/requirements_test.py

@@ -2,11 +2,11 @@
 
 import pandas as pd
 import pytest
+from aqueduct.error import AqueductError, InvalidUserArgumentException
 from transformers_model.model import sentiment_prediction_using_transformers
 from utils import SENTIMENT_SQL_QUERY, get_integration_name
 
 from aqueduct import op
-from aqueduct.error import AqueductError, InvalidUserArgumentException
 
 INVALID_REQUIREMENTS_PATH = "~/random.txt"
 VALID_REQUIREMENTS_PATH = "transformers_model/requirements.txt"

integration_tests/sdk/aqueduct_tests/checks_test.py

@@ -1,7 +1,5 @@
 import pandas as pd
 import pytest
-
-from aqueduct import check
 from aqueduct.constants.enums import CheckSeverity
 from aqueduct.error import (
     AqueductError,
@@ -10,6 +8,8 @@
     InvalidUserActionException,
 )
 
+from aqueduct import check
+
 from ..shared.data_objects import DataObject
 from ..shared.flow_helpers import publish_flow_test
 from .extract import extract

integration_tests/sdk/aqueduct_tests/conftest.py

@@ -1,5 +1,4 @@
 import pytest
-
 from aqueduct.constants.enums import ServiceType
 
 from .data_validator import DataValidator

integration_tests/sdk/aqueduct_tests/data_validator.py

@@ -2,9 +2,9 @@
 from typing import Any
 
 import pandas as pd
+from aqueduct.models.integration import Integration
 
 from aqueduct import Client, Flow
-from aqueduct.models.integration import Integration
 
 from ..shared.globals import artifact_id_to_saved_identifier
 from ..shared.validation import fetch_and_validate_saved_object_identifier

integration_tests/sdk/aqueduct_tests/delete_workflow_test.py

@@ -1,10 +1,10 @@
 import pandas as pd
 import pytest
+from aqueduct.constants.enums import LoadUpdateMode
+from aqueduct.error import InvalidRequestError, InvalidUserArgumentException
 
 import aqueduct
 from aqueduct import op
-from aqueduct.constants.enums import LoadUpdateMode
-from aqueduct.error import InvalidRequestError, InvalidUserArgumentException
 
 from ..shared.data_objects import DataObject
 from ..shared.flow_helpers import publish_flow_test

integration_tests/sdk/aqueduct_tests/error_handling_test.py

@@ -1,8 +1,8 @@
 import pytest
+from aqueduct.error import AqueductError, InvalidUserArgumentException
 
 import aqueduct
 from aqueduct import op
-from aqueduct.error import AqueductError, InvalidUserArgumentException
 
 from ..shared.data_objects import DataObject
 from .extract import extract

integration_tests/sdk/aqueduct_tests/external_compute_test.py

@@ -1,9 +1,9 @@
 import platform
 
 import pytest
+from aqueduct.error import AqueductError
 
 from aqueduct import check, global_config, op
-from aqueduct.error import AqueductError
 
 from ..shared.data_objects import DataObject
 from ..shared.flow_helpers import publish_flow_test

integration_tests/sdk/aqueduct_tests/extract.py

@@ -4,6 +4,7 @@
 from aqueduct.constants.enums import ArtifactType
 from aqueduct.integrations.s3_integration import S3Integration
 from aqueduct.integrations.sql_integration import RelationalDBIntegration
+
 from sdk.shared.data_objects import DataObject
 
 

integration_tests/sdk/aqueduct_tests/flow_test.py

@@ -3,11 +3,11 @@
 
 import pandas as pd
 import pytest
+from aqueduct.constants.enums import ExecutionStatus
+from aqueduct.error import InvalidRequestError, InvalidUserArgumentException
 
 import aqueduct
 from aqueduct import check, metric, op
-from aqueduct.constants.enums import ExecutionStatus
-from aqueduct.error import InvalidRequestError, InvalidUserArgumentException
 
 from ..shared.data_objects import DataObject
 from ..shared.flow_helpers import publish_flow_test, trigger_flow_test, wait_for_flow_runs

integration_tests/sdk/aqueduct_tests/integration_test.py

@@ -1,7 +1,4 @@
 import pytest
-from pydantic import ValidationError
-
-from aqueduct import global_config
 from aqueduct.constants.enums import ServiceType
 from aqueduct.error import (
     AqueductError,
@@ -11,6 +8,9 @@
     InvalidUserArgumentException,
 )
 from aqueduct.integrations.connect_config import K8sConfig
+from pydantic import ValidationError
+
+from aqueduct import global_config
 
 from ..shared.data_objects import DataObject
 from .extract import extract

integration_tests/sdk/aqueduct_tests/lazy_execution_test.py

@@ -2,14 +2,14 @@
 
 import pandas as pd
 import pytest
-
-from aqueduct import check, global_config, metric, op
 from aqueduct.artifacts.bool_artifact import BoolArtifact
 from aqueduct.artifacts.generic_artifact import GenericArtifact
 from aqueduct.artifacts.numeric_artifact import NumericArtifact
 from aqueduct.constants.enums import ArtifactType
 from aqueduct.error import InvalidUserArgumentException
 
+from aqueduct import check, global_config, metric, op
+
 from ..shared.data_objects import DataObject
 from ..shared.flow_helpers import publish_flow_test
 from .extract import extract

integration_tests/sdk/aqueduct_tests/metrics_test.py

@@ -1,8 +1,8 @@
 import pandas as pd
 import pytest
+from aqueduct.error import AqueductError, ArtifactNotFoundException
 
 from aqueduct import metric
-from aqueduct.error import AqueductError, ArtifactNotFoundException
 
 from ..shared.data_objects import DataObject
 from ..shared.flow_helpers import publish_flow_test

integration_tests/sdk/aqueduct_tests/multiple_output_test.py

@@ -1,7 +1,7 @@
 import pytest
+from aqueduct.error import AqueductError
 
 from aqueduct import op
-from aqueduct.error import AqueductError
 
 from ..shared.flow_helpers import publish_flow_test
 

integration_tests/sdk/aqueduct_tests/naming_test.py

@@ -1,7 +1,7 @@
 import pytest
+from aqueduct.error import ArtifactNotFoundException, InvalidUserActionException
 
 from aqueduct import check, metric, op
-from aqueduct.error import ArtifactNotFoundException, InvalidUserActionException
 
 from ..shared.data_objects import DataObject
 from ..shared.flow_helpers import publish_flow_test

integration_tests/sdk/aqueduct_tests/operator_hierarchy_test.py

@@ -1,7 +1,7 @@
 import pytest
+from aqueduct.error import InvalidUserActionException
 
 from aqueduct import check, metric, op
-from aqueduct.error import InvalidUserActionException
 
 from ..shared.data_objects import DataObject
 from .extract import extract

integration_tests/sdk/aqueduct_tests/operator_test.py

@@ -1,9 +1,9 @@
 import pandas as pd
 import pytest
-
-from aqueduct import op
 from aqueduct.decorator import to_operator
 from aqueduct.error import ArtifactNotFoundException
+
+from aqueduct import op
 from sdk.aqueduct_tests.test_function import dummy_sentiment_model_function
 
 from ..shared.data_objects import DataObject

integration_tests/sdk/aqueduct_tests/param_test.py

@@ -4,10 +4,6 @@
 
 import pandas as pd
 import pytest
-from pandas._testing import assert_frame_equal
-from PIL import Image
-
-from aqueduct import metric, op
 from aqueduct.artifacts.generic_artifact import GenericArtifact
 from aqueduct.artifacts.numeric_artifact import NumericArtifact
 from aqueduct.constants.enums import ArtifactType, ExecutionStatus
@@ -17,6 +13,10 @@
     InvalidUserActionException,
     InvalidUserArgumentException,
 )
+from pandas._testing import assert_frame_equal
+from PIL import Image
+
+from aqueduct import metric, op
 
 from ..shared.data_objects import DataObject
 from ..shared.flow_helpers import publish_flow_test, trigger_flow_test

integration_tests/sdk/aqueduct_tests/preview_test.py

@@ -1,8 +1,8 @@
 import pandas as pd
 import pytest
+from aqueduct.error import AqueductError, InvalidDependencyFilePath, InvalidFunctionException
 
 from aqueduct import global_config, op
-from aqueduct.error import AqueductError, InvalidDependencyFilePath, InvalidFunctionException
 
 from ..shared.data_objects import DataObject
 from .extract import extract

integration_tests/sdk/aqueduct_tests/resources_test.py

@@ -1,11 +1,11 @@
 from os import cpu_count
 
 import pytest
-
-from aqueduct import global_config, op
 from aqueduct.constants.enums import ExecutionStatus, ServiceType
 from aqueduct.error import AqueductError, InvalidUserArgumentException
 
+from aqueduct import global_config, op
+
 from ..shared.flow_helpers import publish_flow_test
 
 

integration_tests/sdk/aqueduct_tests/save.py

@@ -1,12 +1,13 @@
 from typing import Optional
 
-from aqueduct import LoadUpdateMode
 from aqueduct.artifacts.table_artifact import TableArtifact
 from aqueduct.constants.enums import S3TableFormat
 from aqueduct.integrations.mongodb_integration import MongoDBIntegration
 from aqueduct.integrations.s3_integration import S3Integration
 from aqueduct.integrations.sql_integration import RelationalDBIntegration
 
+from aqueduct import LoadUpdateMode
+
 from ..shared.globals import artifact_id_to_saved_identifier
 from ..shared.naming import generate_table_name
 

integration_tests/sdk/aqueduct_tests/spark_test.py

@@ -1,11 +1,11 @@
 from os import cpu_count
 
 import pytest
-
-from aqueduct import global_config, op
 from aqueduct.constants.enums import ExecutionStatus, ServiceType
 from aqueduct.error import AqueductError, InvalidUserArgumentException
 
+from aqueduct import global_config, op
+
 from ..shared.data_objects import DataObject
 from ..shared.flow_helpers import publish_flow_test
 from .extract import extract

integration_tests/sdk/aqueduct_tests/type_enforcement_test.py

@@ -1,11 +1,11 @@
 from typing import Union
 
 import pytest
-
-from aqueduct import op
 from aqueduct.constants.enums import ExecutionStatus
 from aqueduct.error import AqueductError
 
+from aqueduct import op
+
 from ..shared.flow_helpers import publish_flow_test, trigger_flow_test
 
 

integration_tests/sdk/conftest.py

@@ -1,10 +1,10 @@
 from typing import Optional
 
 import pytest
-
-from aqueduct import Client, global_config, globals
 from aqueduct.constants.enums import ServiceType
 from aqueduct.models.dag import DAG, Metadata
+
+from aqueduct import Client, global_config, globals
 from sdk.setup_integration import (
     get_aqueduct_config,
     get_artifact_store_name,

integration_tests/sdk/data_integration_tests/athena_test.py

@@ -1,10 +1,10 @@
 import pandas as pd
 import pytest
-
-from aqueduct import LoadUpdateMode, metric, op
 from aqueduct.error import InvalidUserActionException, InvalidUserArgumentException
 from aqueduct.integrations.sql_integration import RelationalDBIntegration
 
+from aqueduct import LoadUpdateMode, metric, op
+
 from ..shared.naming import generate_table_name
 from .save import save
 

integration_tests/sdk/data_integration_tests/conftest.py

@@ -5,6 +5,7 @@
 # Maps the test files in this directory to the allowed data integrations for that file.
 # If a disallowed data integration is used, all tests in the file will be skipped.
 from aqueduct.constants.enums import ServiceType
+
 from sdk.data_integration_tests.flow_manager import FlowManager
 
 allowed_data_integrations_by_file = {

integration_tests/sdk/data_integration_tests/flow_manager.py

@@ -1,8 +1,9 @@
 from typing import Any, Callable, Dict, List, Optional, Union
 
-from aqueduct import Client, Flow
 from aqueduct.artifacts.base_artifact import BaseArtifact
 from aqueduct.constants.enums import ExecutionStatus
+
+from aqueduct import Client, Flow
 from sdk.shared.flow_helpers import publish_flow_test, trigger_flow_test
 
 

integration_tests/sdk/data_integration_tests/mongo_db_data_validator.py

@@ -2,9 +2,9 @@
 from typing import Any
 
 import pandas as pd
+from aqueduct.integrations.mongodb_integration import MongoDBIntegration
 
 from aqueduct import Client, Flow
-from aqueduct.integrations.mongodb_integration import MongoDBIntegration
 from sdk.data_integration_tests.relational_data_validator import RelationalDataValidator
 
 from ..shared.validation import fetch_and_validate_saved_object_identifier

integration_tests/sdk/data_integration_tests/mongo_db_test.py

@@ -2,10 +2,10 @@
 
 import pandas as pd
 import pytest
-
-from aqueduct import LoadUpdateMode, op
 from aqueduct.error import AqueductError
 from aqueduct.integrations.mongodb_integration import MongoDBIntegration
+
+from aqueduct import LoadUpdateMode, op
 from sdk.data_integration_tests.flow_manager import FlowManager
 from sdk.data_integration_tests.mongo_db_data_validator import MongoDBDataValidator
 from sdk.data_integration_tests.save import save