Add --standalone-dag-processor option to start-airflow command (a…

…pache#35139)

As part of AIP-44 implementation we are going to test more secure
combination of airflow so adding standalone dag-processor to breeze
will make it easier.

Dockerfile.ci

@@ -760,6 +760,12 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
     echo "  * ${COLOR_BLUE}Airflow core SQL connection:${COLOR_RESET} ${AIRFLOW__CORE__SQL_ALCHEMY_CONN:=}"
     echo
 
+    if [[ ${STANDALONE_DAG_PROCESSOR=} == "true" ]]; then
+        echo
+        echo "${COLOR_BLUE}Running forcing scheduler/standalone_dag_processor to be True${COLOR_RESET}"
+        echo
+        export AIRFLOW__SCHEDULER__STANDALONE_DAG_PROCESSOR=True
+    fi
     RUN_TESTS=${RUN_TESTS:="false"}
     CI=${CI:="false"}
     USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION:=""}"

dev/breeze/src/airflow_breeze/commands/developer_commands.py

@@ -71,6 +71,7 @@
     option_platform_single,
     option_postgres_version,
     option_python,
+    option_standalone_dag_processor,
     option_upgrade_boto,
     option_use_airflow_version,
     option_use_packages_from_dist,
@@ -173,6 +174,7 @@ def run(self):
 @option_executor
 @option_celery_broker
 @option_celery_flower
+@option_standalone_dag_processor
 @click.argument("extra-args", nargs=-1, type=click.UNPROCESSED)
 def shell(
     python: str,
@@ -203,6 +205,7 @@ def shell(
     extra_args: tuple,
     upgrade_boto: bool,
     downgrade_sqlalchemy: bool,
+    standalone_dag_processor: bool,
 ):
     """Enter breeze environment. this is the default command use when no other is selected."""
     if get_verbose() or get_dry_run():
@@ -242,6 +245,7 @@ def shell(
         celery_flower=celery_flower,
         upgrade_boto=upgrade_boto,
         downgrade_sqlalchemy=downgrade_sqlalchemy,
+        standalone_dag_processor=standalone_dag_processor,
     )
     sys.exit(result.returncode)
 
@@ -287,6 +291,7 @@ def shell(
 @option_executor
 @option_celery_broker
 @option_celery_flower
+@option_standalone_dag_processor
 def start_airflow(
     python: str,
     backend: str,
@@ -315,6 +320,7 @@ def start_airflow(
     executor: str,
     celery_broker: str,
     celery_flower: bool,
+    standalone_dag_processor: bool,
 ):
     """
     Enter breeze environment and starts all Airflow components in the tmux session.
@@ -359,6 +365,7 @@ def start_airflow(
         executor=executor,
         celery_broker=celery_broker,
         celery_flower=celery_flower,
+        standalone_dag_processor=standalone_dag_processor,
     )
     sys.exit(result.returncode)
 

dev/breeze/src/airflow_breeze/commands/developer_commands_config.py

@@ -40,6 +40,7 @@
                 "--mysql-version",
                 "--mssql-version",
                 "--integration",
+                "--standalone-dag-processor",
                 "--forward-credentials",
                 "--db-reset",
                 "--max-time",
@@ -71,6 +72,12 @@
                 "--celery-flower",
             ],
         },
+        {
+            "name": "Airflow execution modes",
+            "options": [
+                "--standalone-dag-processor",
+            ],
+        },
         {
             "name": "Building image before entering shell",
             "options": [
@@ -140,6 +147,12 @@
                 "--celery-flower",
             ],
         },
+        {
+            "name": "Airflow execution modes",
+            "options": [
+                "--standalone-dag-processor",
+            ],
+        },
         {
             "name": "Asset compilation options",
             "options": [

dev/breeze/src/airflow_breeze/commands/main_command.py

@@ -41,6 +41,7 @@
     option_mysql_version,
     option_postgres_version,
     option_python,
+    option_standalone_dag_processor,
     option_verbose,
 )
 from airflow_breeze.utils.confirm import Answer, user_confirm
@@ -107,6 +108,7 @@ def get_command(self, ctx: Context, cmd_name: str):
 @option_mysql_version
 @option_mssql_version
 @option_integration
+@option_standalone_dag_processor
 @option_forward_credentials
 @option_db_reset
 @option_max_time

dev/breeze/src/airflow_breeze/params/shell_params.py

@@ -127,6 +127,7 @@ class ShellParams:
     only_min_version_update: bool = False
     regenerate_missing_docs: bool = False
     skip_provider_dependencies_check: bool = False
+    standalone_dag_processor: bool = False
 
     def clone_with_test(self, test_type: str) -> ShellParams:
         new_params = deepcopy(self)

dev/breeze/src/airflow_breeze/utils/common_options.py

@@ -575,6 +575,12 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option,
     show_default=True,
 )
 option_celery_flower = click.option("--celery-flower", help="Start celery flower", is_flag=True)
+option_standalone_dag_processor = click.option(
+    "--standalone-dag-processor",
+    help="Run standalone dag processor for start-airflow.",
+    is_flag=True,
+    envvar="STANDALONE_DAG_PROCESSOR",
+)
 option_install_selected_providers = click.option(
     "--install-selected-providers",
     help="Comma-separated list of providers selected to be installed (implies --use-packages-from-dist).",

dev/breeze/src/airflow_breeze/utils/docker_command_utils.py

@@ -662,6 +662,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
     "USE_PACKAGES_FROM_DIST": "use_packages_from_dist",
     "VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi",
     "CELERY_FLOWER": "celery_flower",
+    "STANDALONE_DAG_PROCESSOR": "standalone_dag_processor",
 }
 
 DOCKER_VARIABLE_CONSTANTS = {

images/breeze/output-commands-hash.txt

@@ -1,7 +1,7 @@
 # This file is automatically generated by pre-commit. If you have a conflict with this file
 # Please do not solve it but run `breeze setup regenerate-command-images`.
 # This command should fix the conflict and regenerate help images that you have conflict with.
-main:c97d8a728bb02c2be73002c4b39d0829
+main:4098f8e87f8aab37816dfe7b91f1b675
 build-docs:1b1848521361fc7fa711409cf9132108
 ci:find-backtracking-candidates:17fe56b867a745e5032a08dfcd3f73ee
 ci:fix-ownership:3e5a73533cc96045e72cb258783cfc96
@@ -62,8 +62,8 @@ setup:regenerate-command-images:a228071206e00b6004ede2e69b8f4a6c
 setup:self-upgrade:4af905a147fcd6670a0e33d3d369a94b
 setup:version:be116d90a21c2afe01087f7609774e1e
 setup:c569719a4fc7be726556a266e447c871
-shell:a45ee80a2937ec3c9741a4558f312d7a
-start-airflow:b9af8a9cd1e3d353aadcd0ccbc7a06be
+shell:676e7d054d7ce079301018a98f406545
+start-airflow:9a5668007f0acf1b954456c12d8b4051
 static-checks:19926b8fcea5784b28d4a0d99865363c
 testing:docker-compose-tests:fd154a058082fcfda12eb877a9a89338
 testing:helm-tests:0669be17b744ba057adbf38681bd8e68