diff --git a/Dockerfile.ci b/Dockerfile.ci index 5a73b81e21a28..10355b5f046db 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -760,6 +760,12 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then echo " * ${COLOR_BLUE}Airflow core SQL connection:${COLOR_RESET} ${AIRFLOW__CORE__SQL_ALCHEMY_CONN:=}" echo + if [[ ${STANDALONE_DAG_PROCESSOR=} == "true" ]]; then + echo + echo "${COLOR_BLUE}Running forcing scheduler/standalone_dag_processor to be True${COLOR_RESET}" + echo + export AIRFLOW__SCHEDULER__STANDALONE_DAG_PROCESSOR=True + fi RUN_TESTS=${RUN_TESTS:="false"} CI=${CI:="false"} USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION:=""}" diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index 7b71e3e70705d..5829cc471c19d 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -71,6 +71,7 @@ option_platform_single, option_postgres_version, option_python, + option_standalone_dag_processor, option_upgrade_boto, option_use_airflow_version, option_use_packages_from_dist, @@ -173,6 +174,7 @@ def run(self): @option_executor @option_celery_broker @option_celery_flower +@option_standalone_dag_processor @click.argument("extra-args", nargs=-1, type=click.UNPROCESSED) def shell( python: str, @@ -203,6 +205,7 @@ def shell( extra_args: tuple, upgrade_boto: bool, downgrade_sqlalchemy: bool, + standalone_dag_processor: bool, ): """Enter breeze environment. this is the default command use when no other is selected.""" if get_verbose() or get_dry_run(): @@ -242,6 +245,7 @@ def shell( celery_flower=celery_flower, upgrade_boto=upgrade_boto, downgrade_sqlalchemy=downgrade_sqlalchemy, + standalone_dag_processor=standalone_dag_processor, ) sys.exit(result.returncode) @@ -287,6 +291,7 @@ def shell( @option_executor @option_celery_broker @option_celery_flower +@option_standalone_dag_processor def start_airflow( python: str, backend: str, @@ -315,6 +320,7 @@ def start_airflow( executor: str, celery_broker: str, celery_flower: bool, + standalone_dag_processor: bool, ): """ Enter breeze environment and starts all Airflow components in the tmux session. @@ -359,6 +365,7 @@ def start_airflow( executor=executor, celery_broker=celery_broker, celery_flower=celery_flower, + standalone_dag_processor=standalone_dag_processor, ) sys.exit(result.returncode) diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py index 9ef4f8ca96d63..e06adb5665371 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py @@ -40,6 +40,7 @@ "--mysql-version", "--mssql-version", "--integration", + "--standalone-dag-processor", "--forward-credentials", "--db-reset", "--max-time", @@ -71,6 +72,12 @@ "--celery-flower", ], }, + { + "name": "Airflow execution modes", + "options": [ + "--standalone-dag-processor", + ], + }, { "name": "Building image before entering shell", "options": [ @@ -140,6 +147,12 @@ "--celery-flower", ], }, + { + "name": "Airflow execution modes", + "options": [ + "--standalone-dag-processor", + ], + }, { "name": "Asset compilation options", "options": [ diff --git a/dev/breeze/src/airflow_breeze/commands/main_command.py b/dev/breeze/src/airflow_breeze/commands/main_command.py index 8b1a115a231b7..927ab89133f95 100644 --- a/dev/breeze/src/airflow_breeze/commands/main_command.py +++ b/dev/breeze/src/airflow_breeze/commands/main_command.py @@ -41,6 +41,7 @@ option_mysql_version, option_postgres_version, option_python, + option_standalone_dag_processor, option_verbose, ) from airflow_breeze.utils.confirm import Answer, user_confirm @@ -107,6 +108,7 @@ def get_command(self, ctx: Context, cmd_name: str): @option_mysql_version @option_mssql_version @option_integration +@option_standalone_dag_processor @option_forward_credentials @option_db_reset @option_max_time diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index b2240a42c991c..6ca36e58b5dfc 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -127,6 +127,7 @@ class ShellParams: only_min_version_update: bool = False regenerate_missing_docs: bool = False skip_provider_dependencies_check: bool = False + standalone_dag_processor: bool = False def clone_with_test(self, test_type: str) -> ShellParams: new_params = deepcopy(self) diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index 7353b4de7e171..6165ae69dfb72 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -575,6 +575,12 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, show_default=True, ) option_celery_flower = click.option("--celery-flower", help="Start celery flower", is_flag=True) +option_standalone_dag_processor = click.option( + "--standalone-dag-processor", + help="Run standalone dag processor for start-airflow.", + is_flag=True, + envvar="STANDALONE_DAG_PROCESSOR", +) option_install_selected_providers = click.option( "--install-selected-providers", help="Comma-separated list of providers selected to be installed (implies --use-packages-from-dist).", diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index fb9b66da50271..39bf0798720d4 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -662,6 +662,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None: "USE_PACKAGES_FROM_DIST": "use_packages_from_dist", "VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi", "CELERY_FLOWER": "celery_flower", + "STANDALONE_DAG_PROCESSOR": "standalone_dag_processor", } DOCKER_VARIABLE_CONSTANTS = { diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 96893d59ea033..4c1799c333a29 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -1,7 +1,7 @@ # This file is automatically generated by pre-commit. If you have a conflict with this file # Please do not solve it but run `breeze setup regenerate-command-images`. # This command should fix the conflict and regenerate help images that you have conflict with. -main:c97d8a728bb02c2be73002c4b39d0829 +main:4098f8e87f8aab37816dfe7b91f1b675 build-docs:1b1848521361fc7fa711409cf9132108 ci:find-backtracking-candidates:17fe56b867a745e5032a08dfcd3f73ee ci:fix-ownership:3e5a73533cc96045e72cb258783cfc96 @@ -62,8 +62,8 @@ setup:regenerate-command-images:a228071206e00b6004ede2e69b8f4a6c setup:self-upgrade:4af905a147fcd6670a0e33d3d369a94b setup:version:be116d90a21c2afe01087f7609774e1e setup:c569719a4fc7be726556a266e447c871 -shell:a45ee80a2937ec3c9741a4558f312d7a -start-airflow:b9af8a9cd1e3d353aadcd0ccbc7a06be +shell:676e7d054d7ce079301018a98f406545 +start-airflow:9a5668007f0acf1b954456c12d8b4051 static-checks:19926b8fcea5784b28d4a0d99865363c testing:docker-compose-tests:fd154a058082fcfda12eb877a9a89338 testing:helm-tests:0669be17b744ba057adbf38681bd8e68 diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index cfcf125e4991e..1ff44a478f0d6 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -1,4 +1,4 @@ - +