Use Python 3.9 as target version for Ruff & Black rules (apache#44298)

.pre-commit-config.yaml

@@ -1189,7 +1189,13 @@ repos:
         files: \.py$
         exclude: |
           (?x)
+          ^airflow/configuration.py$ |
+          ^airflow/metrics/validators.py$ |
+          ^airflow/models/dag.py$ |
+          ^airflow/serialization/serde.py$ |
+          ^airflow/utils/file.py$ |
           ^airflow/utils/helpers.py$ |
+          ^airflow/utils/log/secrets_masker.py$ |
           ^providers/src/airflow/providers/ |
           ^(providers/)?tests/ |
           task_sdk/src/airflow/sdk/definitions/dag.py$ |

airflow/api/common/mark_tasks.py

@@ -19,7 +19,8 @@
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Collection, Iterable
+from collections.abc import Collection, Iterable
+from typing import TYPE_CHECKING
 
 from sqlalchemy import and_, or_, select
 from sqlalchemy.orm import lazyload

airflow/api_connexion/endpoints/dag_endpoint.py

@@ -16,8 +16,9 @@
 # under the License.
 from __future__ import annotations
 
+from collections.abc import Collection
 from http import HTTPStatus
-from typing import TYPE_CHECKING, Collection
+from typing import TYPE_CHECKING
 
 from connexion import NoContent
 from flask import g, request

airflow/api_connexion/endpoints/dag_parsing.py

@@ -16,8 +16,9 @@
 # under the License.
 from __future__ import annotations
 
+from collections.abc import Sequence
 from http import HTTPStatus
-from typing import TYPE_CHECKING, Sequence
+from typing import TYPE_CHECKING
 
 from flask import Response, current_app
 from itsdangerous import BadSignature, URLSafeSerializer

airflow/api_connexion/endpoints/dag_run_endpoint.py

@@ -16,8 +16,9 @@
 # under the License.
 from __future__ import annotations
 
+from collections.abc import Collection
 from http import HTTPStatus
-from typing import TYPE_CHECKING, Collection
+from typing import TYPE_CHECKING
 
 import pendulum
 from connexion import NoContent

airflow/api_connexion/endpoints/dag_source_endpoint.py

@@ -16,8 +16,9 @@
 # under the License.
 from __future__ import annotations
 
+from collections.abc import Sequence
 from http import HTTPStatus
-from typing import TYPE_CHECKING, Sequence
+from typing import TYPE_CHECKING
 
 from flask import Response, request
 from sqlalchemy import select

airflow/api_connexion/endpoints/import_error_endpoint.py

@@ -16,7 +16,8 @@
 # under the License.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Sequence
+from collections.abc import Sequence
+from typing import TYPE_CHECKING
 
 from sqlalchemy import func, select
 

airflow/api_connexion/endpoints/pool_endpoint.py

@@ -119,8 +119,8 @@ def patch_pool(
         update_mask = [i.strip() for i in update_mask]
         _patch_body = {}
         try:
-            # MyPy infers a List[Optional[str]]  type here but it should be a List[str]
-            # there is no way field is None here (UpdateMask is a List[str])
+            # MyPy infers a list[Optional[str]]  type here but it should be a list[str]
+            # there is no way field is None here (UpdateMask is a list[str])
             # so if pool_schema.declared_fields[field].attribute is None file is returned
             update_mask = [
                 (

airflow/api_connexion/endpoints/request_dict.py

@@ -16,7 +16,8 @@
 # under the License.
 from __future__ import annotations
 
-from typing import Any, Mapping, cast
+from collections.abc import Mapping
+from typing import Any, cast
 
 
 def get_json_request_dict() -> Mapping[str, Any]:

airflow/api_connexion/endpoints/task_instance_endpoint.py

@@ -16,7 +16,8 @@
 # under the License.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, Iterable, Sequence, TypeVar
+from collections.abc import Iterable, Sequence
+from typing import TYPE_CHECKING, Any, TypeVar
 
 from flask import g
 from marshmallow import ValidationError

airflow/api_connexion/endpoints/update_mask.py

@@ -16,7 +16,8 @@
 # under the License.
 from __future__ import annotations
 
-from typing import Any, Mapping, Sequence
+from collections.abc import Mapping, Sequence
+from typing import Any
 
 from airflow.api_connexion.exceptions import BadRequest
 

airflow/api_connexion/parameters.py

@@ -17,8 +17,9 @@
 from __future__ import annotations
 
 import logging
+from collections.abc import Container
 from functools import wraps
-from typing import TYPE_CHECKING, Any, Callable, Container, TypeVar, cast
+from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast
 
 from pendulum.parsing import ParserError
 from sqlalchemy import text

airflow/api_connexion/types.py

@@ -16,13 +16,14 @@
 # under the License.
 from __future__ import annotations
 
-from typing import Any, Mapping, Optional, Sequence, Tuple, Union
+from collections.abc import Mapping, Sequence
+from typing import Any, Optional, Union
 
 from flask import Response
 
 APIResponse = Union[
     Response,
-    Tuple[object, int],  # For '(NoContent, 201)'.
+    tuple[object, int],  # For '(NoContent, 201)'.
     Mapping[str, Any],  # JSON.
 ]
 

airflow/api_fastapi/common/db/common.py

@@ -22,7 +22,8 @@
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Literal, Sequence, overload
+from collections.abc import Sequence
+from typing import TYPE_CHECKING, Literal, overload
 
 from sqlalchemy.ext.asyncio import AsyncSession
 

airflow/api_fastapi/common/headers.py

@@ -16,8 +16,9 @@
 # under the License.
 from __future__ import annotations
 
+from typing import Annotated
+
 from fastapi import Depends, Header, HTTPException, status
-from typing_extensions import Annotated
 
 from airflow.api_fastapi.common.types import Mimetype
 

airflow/api_fastapi/common/parameters.py

@@ -18,15 +18,14 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
+from collections.abc import Iterable
 from datetime import datetime
 from typing import (
     TYPE_CHECKING,
     Annotated,
     Any,
     Callable,
     Generic,
-    Iterable,
-    List,
     Optional,
     TypeVar,
     Union,
@@ -288,7 +287,7 @@ def inner(order_by: str = default or self.get_primary_key_string()) -> SortParam
         return inner
 
 
-class _TagsFilter(BaseParam[List[str]]):
+class _TagsFilter(BaseParam[list[str]]):
     """Filter on tags."""
 
     def to_orm(self, select: Select) -> Select:
@@ -305,7 +304,7 @@ def depends(self, tags: list[str] = Query(default_factory=list)) -> _TagsFilter:
         return self.set_value(tags)
 
 
-class _OwnersFilter(BaseParam[List[str]]):
+class _OwnersFilter(BaseParam[list[str]]):
     """Filter on owners."""
 
     def to_orm(self, select: Select) -> Select:
@@ -322,7 +321,7 @@ def depends(self, owners: list[str] = Query(default_factory=list)) -> _OwnersFil
         return self.set_value(owners)
 
 
-class DagRunStateFilter(BaseParam[List[Optional[DagRunState]]]):
+class DagRunStateFilter(BaseParam[list[Optional[DagRunState]]]):
     """Filter on Dag Run state."""
 
     def to_orm(self, select: Select) -> Select:
@@ -352,7 +351,7 @@ def depends(self, state: list[str] = Query(default_factory=list)) -> DagRunState
         return self.set_value(states)
 
 
-class TIStateFilter(BaseParam[List[Optional[TaskInstanceState]]]):
+class TIStateFilter(BaseParam[list[Optional[TaskInstanceState]]]):
     """Filter on task instance state."""
 
     def to_orm(self, select: Select) -> Select:
@@ -376,7 +375,7 @@ def depends(self, state: list[str] = Query(default_factory=list)) -> TIStateFilt
         return self.set_value(states)
 
 
-class TIPoolFilter(BaseParam[List[str]]):
+class TIPoolFilter(BaseParam[list[str]]):
     """Filter on task instance pool."""
 
     def to_orm(self, select: Select) -> Select:
@@ -393,7 +392,7 @@ def depends(self, pool: list[str] = Query(default_factory=list)) -> TIPoolFilter
         return self.set_value(pool)
 
 
-class TIQueueFilter(BaseParam[List[str]]):
+class TIQueueFilter(BaseParam[list[str]]):
     """Filter on task instance queue."""
 
     def to_orm(self, select: Select) -> Select:
@@ -410,7 +409,7 @@ def depends(self, queue: list[str] = Query(default_factory=list)) -> TIQueueFilt
         return self.set_value(queue)
 
 
-class TIExecutorFilter(BaseParam[List[str]]):
+class TIExecutorFilter(BaseParam[list[str]]):
     """Filter on task instance executor."""
 
     def to_orm(self, select: Select) -> Select:

airflow/api_fastapi/common/router.py

@@ -17,8 +17,9 @@
 
 from __future__ import annotations
 
+from collections.abc import Sequence
 from enum import Enum
-from typing import Any, Callable, Sequence
+from typing import Any, Callable
 
 from fastapi import APIRouter, params
 from fastapi.datastructures import Default

airflow/api_fastapi/core_api/datamodels/dags.py

@@ -18,8 +18,9 @@
 from __future__ import annotations
 
 from collections import abc
+from collections.abc import Iterable
 from datetime import datetime, timedelta
-from typing import Any, Iterable
+from typing import Any
 
 from itsdangerous import URLSafeSerializer
 from pendulum.tz.timezone import FixedTimezone, Timezone

airflow/api_fastapi/core_api/routes/public/xcom.py

@@ -17,11 +17,11 @@
 from __future__ import annotations
 
 import copy
+from typing import Annotated
 
 from fastapi import Depends, HTTPException, Query, status
 from sqlalchemy import and_, select
 from sqlalchemy.orm import Session
-from typing_extensions import Annotated
 
 from airflow.api_fastapi.common.db.common import get_session
 from airflow.api_fastapi.common.router import AirflowRouter

airflow/api_fastapi/core_api/security.py

@@ -17,12 +17,11 @@
 from __future__ import annotations
 
 from functools import cache
-from typing import Any, Callable
+from typing import Annotated, Any, Callable
 
 from fastapi import Depends, HTTPException
 from fastapi.security import OAuth2PasswordBearer
 from jwt import InvalidTokenError
-from typing_extensions import Annotated
 
 from airflow.api_fastapi.app import get_auth_manager
 from airflow.auth.managers.base_auth_manager import ResourceMethod

airflow/auth/managers/base_auth_manager.py

@@ -18,8 +18,9 @@
 from __future__ import annotations
 
 from abc import abstractmethod
+from collections.abc import Container, Sequence
 from functools import cached_property
-from typing import TYPE_CHECKING, Any, Container, Generic, Literal, Sequence, TypeVar
+from typing import TYPE_CHECKING, Any, Generic, Literal, TypeVar
 
 from flask_appbuilder.menu import MenuItem
 from sqlalchemy import select

airflow/cli/cli_config.py

@@ -24,7 +24,8 @@
 import json
 import os
 import textwrap
-from typing import Callable, Iterable, NamedTuple, Union
+from collections.abc import Iterable
+from typing import Callable, NamedTuple, Union
 
 import lazy_object_proxy
 

airflow/cli/cli_parser.py

@@ -29,8 +29,9 @@
 import sys
 from argparse import Action
 from collections import Counter
-from functools import lru_cache
-from typing import TYPE_CHECKING, Iterable
+from collections.abc import Iterable
+from functools import cache
+from typing import TYPE_CHECKING
 
 import lazy_object_proxy
 from rich_argparse import RawTextRichHelpFormatter, RichHelpFormatter
@@ -138,7 +139,7 @@ def add_argument(self, action: Action) -> None:
         return super().add_argument(action)
 
 
-@lru_cache(maxsize=None)
+@cache
 def get_parser(dag_parser: bool = False) -> argparse.ArgumentParser:
     """Create and returns command line argument parser."""
     parser = DefaultHelpParser(prog="airflow", formatter_class=AirflowHelpFormatter)

airflow/cli/commands/cheat_sheet_command.py

@@ -16,7 +16,8 @@
 # under the License.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Iterable
+from collections.abc import Iterable
+from typing import TYPE_CHECKING
 
 from airflow.cli.cli_parser import GroupCommand, airflow_commands
 from airflow.cli.simple_table import AirflowConsole, SimpleTable

airflow/cli/commands/task_command.py

@@ -26,8 +26,9 @@
 import os
 import sys
 import textwrap
+from collections.abc import Generator
 from contextlib import contextmanager, redirect_stderr, redirect_stdout, suppress
-from typing import TYPE_CHECKING, Generator, Protocol, Union, cast
+from typing import TYPE_CHECKING, Protocol, Union, cast
 
 import pendulum
 from pendulum.parsing.exceptions import ParserError

airflow/cli/commands/triggerer_command.py

@@ -18,10 +18,10 @@
 
 from __future__ import annotations
 
+from collections.abc import Generator
 from contextlib import contextmanager
 from functools import partial
 from multiprocessing import Process
-from typing import Generator
 
 from airflow import settings
 from airflow.cli.commands.daemon_utils import run_command_with_daemon_option

airflow/cli/simple_table.py

@@ -18,7 +18,8 @@
 
 import inspect
 import json
-from typing import TYPE_CHECKING, Any, Callable, Sequence
+from collections.abc import Sequence
+from typing import TYPE_CHECKING, Any, Callable
 
 from rich.box import ASCII_DOUBLE_HEAD
 from rich.console import Console

airflow/cli/utils.py

@@ -19,7 +19,8 @@
 
 import io
 import sys
-from typing import TYPE_CHECKING, Collection
+from collections.abc import Collection
+from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from io import IOBase