Allow to add extra flags passed to pip install when building imag…

…es (apache#25337)

This allows a bit more flexibility when building images
as you can pass arbitrary command line switch to ``pip install``
(for example --pre).

Dockerfile

@@ -317,6 +317,7 @@ function install_airflow_dependencies_from_branch_tip() {
     # are conflicts, this might fail, but it should be fixed in the following installation steps
     set -x
     pip install --root-user-action ignore \
+      ${ADDITIONAL_PIP_INSTALL_FLAGS} \
       "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
       --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true
     # make sure correct PIP version is used
@@ -476,6 +477,7 @@ function install_airflow_and_providers_from_docker_context_files(){
     # force reinstall all airflow + provider package local files with eager upgrade
     set -x
     pip install "${pip_flags[@]}" --root-user-action ignore --upgrade --upgrade-strategy eager \
+        ${ADDITIONAL_PIP_INSTALL_FLAGS} \
         ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} \
         ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
     set +x
@@ -496,7 +498,8 @@ function install_all_other_packages_from_docker_context_files() {
         grep -v apache_airflow | grep -v apache-airflow || true)
     if [[ -n "${reinstalling_other_packages}" ]]; then
         set -x
-        pip install --root-user-action ignore --force-reinstall --no-deps --no-index ${reinstalling_other_packages}
+        pip install ${ADDITIONAL_PIP_INSTALL_FLAGS} \
+            --root-user-action ignore --force-reinstall --no-deps --no-index ${reinstalling_other_packages}
         # make sure correct PIP version is used
         pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
         set -x
@@ -547,6 +550,7 @@ function install_airflow() {
         echo
         # eager upgrade
         pip install --root-user-action ignore --upgrade --upgrade-strategy eager \
+            ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
             ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
         if [[ -n "${AIRFLOW_INSTALL_EDITABLE_FLAG}" ]]; then
@@ -555,6 +559,7 @@ function install_airflow() {
             set -x
             pip uninstall apache-airflow --yes
             pip install --root-user-action ignore ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
+                ${ADDITIONAL_PIP_INSTALL_FLAGS} \
                 "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
             set +x
         fi
@@ -571,12 +576,14 @@ function install_airflow() {
         echo
         set -x
         pip install --root-user-action ignore ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
+            ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
             --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"
         # make sure correct PIP version is used
         pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
         # then upgrade if needed without using constraints to account for new limits in setup.py
         pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
+            ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
             "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
         # make sure correct PIP version is used
@@ -617,6 +624,7 @@ function install_additional_dependencies() {
         echo
         set -x
         pip install --root-user-action ignore --upgrade --upgrade-strategy eager \
+            ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
         # make sure correct PIP version is used
         pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
@@ -631,6 +639,7 @@ function install_additional_dependencies() {
         echo
         set -x
         pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
+            ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             ${ADDITIONAL_PYTHON_DEPS}
         # make sure correct PIP version is used
         pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
@@ -1145,6 +1154,9 @@ RUN if [[ -f /docker-context-files/pip.conf ]]; then \
         cp /docker-context-files/.piprc "${AIRFLOW_USER_HOME_DIR}/.piprc"; \
     fi
 
+# Additional PIP flags passed to all pip install commands except reinstalling pip itself
+ARG ADDITIONAL_PIP_INSTALL_FLAGS=""
+
 ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
     AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \
     INSTALL_PROVIDERS_FROM_SOURCES=${INSTALL_PROVIDERS_FROM_SOURCES} \
@@ -1164,6 +1176,7 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
     PATH=${PATH}:${AIRFLOW_USER_HOME_DIR}/.local/bin \
     AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
     PIP_PROGRESS_BAR=${PIP_PROGRESS_BAR} \
+    ADDITIONAL_PIP_INSTALL_FLAGS=${ADDITIONAL_PIP_INSTALL_FLAGS} \
     AIRFLOW_USER_HOME_DIR=${AIRFLOW_USER_HOME_DIR} \
     AIRFLOW_HOME=${AIRFLOW_HOME} \
     AIRFLOW_UID=${AIRFLOW_UID} \

Dockerfile.ci

@@ -277,6 +277,7 @@ function install_airflow_dependencies_from_branch_tip() {
     # are conflicts, this might fail, but it should be fixed in the following installation steps
     set -x
     pip install --root-user-action ignore \
+      ${ADDITIONAL_PIP_INSTALL_FLAGS} \
       "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
       --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true
     # make sure correct PIP version is used
@@ -416,6 +417,7 @@ function install_airflow() {
         echo
         # eager upgrade
         pip install --root-user-action ignore --upgrade --upgrade-strategy eager \
+            ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
             ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
         if [[ -n "${AIRFLOW_INSTALL_EDITABLE_FLAG}" ]]; then
@@ -424,6 +426,7 @@ function install_airflow() {
             set -x
             pip uninstall apache-airflow --yes
             pip install --root-user-action ignore ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
+                ${ADDITIONAL_PIP_INSTALL_FLAGS} \
                 "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
             set +x
         fi
@@ -440,12 +443,14 @@ function install_airflow() {
         echo
         set -x
         pip install --root-user-action ignore ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
+            ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
             --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"
         # make sure correct PIP version is used
         pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
         # then upgrade if needed without using constraints to account for new limits in setup.py
         pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
+            ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
             "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
         # make sure correct PIP version is used
@@ -486,6 +491,7 @@ function install_additional_dependencies() {
         echo
         set -x
         pip install --root-user-action ignore --upgrade --upgrade-strategy eager \
+            ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
         # make sure correct PIP version is used
         pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
@@ -500,6 +506,7 @@ function install_additional_dependencies() {
         echo
         set -x
         pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
+            ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             ${ADDITIONAL_PYTHON_DEPS}
         # make sure correct PIP version is used
         pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
@@ -1127,6 +1134,9 @@ ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
 
 ARG AIRFLOW_VERSION="2.3.0.dev"
 
+# Additional PIP flags passed to all pip install commands except reinstalling pip itself
+ARG ADDITIONAL_PIP_INSTALL_FLAGS=""
+
 ENV AIRFLOW_REPO=${AIRFLOW_REPO}\
     AIRFLOW_BRANCH=${AIRFLOW_BRANCH} \
     AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}${ADDITIONAL_AIRFLOW_EXTRAS:+,}${ADDITIONAL_AIRFLOW_EXTRAS} \
@@ -1154,6 +1164,7 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\
     AIRFLOW_VERSION_SPECIFICATION="" \
     PIP_NO_CACHE_DIR=${PIP_NO_CACHE_DIR} \
     PIP_PROGRESS_BAR=${PIP_PROGRESS_BAR} \
+    ADDITIONAL_PIP_INSTALL_FLAGS=${ADDITIONAL_PIP_INSTALL_FLAGS} \
     CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY} \
     CASS_DRIVER_NO_CYTHON=${CASS_DRIVER_NO_CYTHON}
 

IMAGES.rst

@@ -375,6 +375,10 @@ The following build arguments (``--build-arg`` in docker build command) can be u
 | ``DEPENDENCIES_EPOCH_NUMBER``            | ``2``                                    | increasing this number will reinstall    |
 |                                          |                                          | all apt dependencies                     |
 +------------------------------------------+------------------------------------------+------------------------------------------+
+| ``ADDITIONAL_PIP_INSTALL_FLAGS``         |                                          | additional ``pip`` flags passed to the   |
+|                                          |                                          | installation commands (except when       |
+|                                          |                                          | reinstalling ``pip`` itself)             |
++------------------------------------------+------------------------------------------+------------------------------------------+
 | ``PIP_NO_CACHE_DIR``                     | ``true``                                 | if true, then no pip cache will be       |
 |                                          |                                          | stored                                   |
 +------------------------------------------+------------------------------------------+------------------------------------------+

dev/breeze/src/airflow_breeze/commands/ci_image_commands.py

@@ -30,6 +30,7 @@
     option_additional_dev_apt_deps,
     option_additional_dev_apt_env,
     option_additional_extras,
+    option_additional_pip_install_flags,
     option_additional_python_deps,
     option_additional_runtime_apt_command,
     option_additional_runtime_apt_deps,
@@ -273,6 +274,7 @@ def start_building(params: BuildCiParams, dry_run: bool, verbose: bool):
 @option_airflow_constraints_mode_ci
 @option_airflow_constraints_reference_build
 @option_tag_as_latest
+@option_additional_pip_install_flags
 def build_image(
     verbose: bool,
     dry_run: bool,

dev/breeze/src/airflow_breeze/commands/production_image_commands.py

@@ -30,6 +30,7 @@
     option_additional_dev_apt_deps,
     option_additional_dev_apt_env,
     option_additional_extras,
+    option_additional_pip_install_flags,
     option_additional_python_deps,
     option_additional_runtime_apt_command,
     option_additional_runtime_apt_deps,
@@ -312,6 +313,7 @@ def run_build_in_parallel(
 @option_runtime_apt_command
 @option_runtime_apt_deps
 @option_tag_as_latest
+@option_additional_pip_install_flags
 def build_prod_image(
     verbose: bool,
     dry_run: bool,

dev/breeze/src/airflow_breeze/params/build_ci_params.py

@@ -71,6 +71,7 @@ def required_image_args(self) -> List[str]:
             "additional_dev_apt_command",
             "additional_dev_apt_deps",
             "additional_dev_apt_env",
+            "additional_pip_install_flags",
             "additional_python_deps",
             "additional_runtime_apt_command",
             "additional_runtime_apt_deps",

dev/breeze/src/airflow_breeze/params/build_prod_params.py

@@ -203,6 +203,7 @@ def required_image_args(self) -> List[str]:
             "additional_dev_apt_command",
             "additional_dev_apt_deps",
             "additional_dev_apt_env",
+            "additional_pip_install_flags",
             "additional_python_deps",
             "additional_runtime_apt_command",
             "additional_runtime_apt_deps",

dev/breeze/src/airflow_breeze/params/common_build_params.py

@@ -41,6 +41,7 @@ class CommonBuildParams:
     additional_runtime_apt_command: str = ""
     additional_runtime_apt_deps: str = ""
     additional_runtime_apt_env: str = ""
+    additional_pip_install_flags: str = ""
     airflow_branch: str = os.environ.get('DEFAULT_BRANCH', AIRFLOW_BRANCH)
     airflow_constraints_location: str = ""
     answer: Optional[str] = None

dev/breeze/src/airflow_breeze/utils/common_options.py

@@ -339,6 +339,12 @@
     is_flag=True,
     envvar='VERIFY_IMAGE',
 )
+option_additional_pip_install_flags = click.option(
+    '--additional-pip-install-flags',
+    help='Additional flags added to `pip install` commands (except reinstalling `pip` itself).',
+    envvar='ADDITIONAL_PIP_INSTALL_FLAGS',
+)
+
 option_install_providers_from_sources = click.option(
     '--install-providers-from-sources',
     help="Install providers from sources when installing.",

docs/docker-stack/build-arg-ref.rst

@@ -47,6 +47,10 @@ Those are the most common arguments that you use when you want to build a custom
 +------------------------------------------+------------------------------------------+---------------------------------------------+
 | ``AIRFLOW_PIP_VERSION``                  | ``22.1.2``                               |  PIP version used.                          |
 +------------------------------------------+------------------------------------------+---------------------------------------------+
+| ``ADDITIONAL_PIP_INSTALL_FLAGS``         |                                          | additional ``pip`` flags passed to the      |
+|                                          |                                          | installation commands (except when          |
+|                                          |                                          | reinstalling ``pip`` itself)                |
++------------------------------------------+------------------------------------------+---------------------------------------------+
 | ``PIP_PROGRESS_BAR``                     | ``on``                                   | Progress bar for PIP installation           |
 +------------------------------------------+------------------------------------------+---------------------------------------------+
 | ``AIRFLOW_UID``                          | ``50000``                                | Airflow user UID.                           |

docs/docker-stack/changelog.rst

@@ -59,6 +59,11 @@ here so that users affected can find the reason for the changes.
 Airflow 2.3
 ~~~~~~~~~~~
 
+* 2.4.0
+
+  * You can specify additional ``pip install`` flags when you build the image via ``ADDITIONAL_PIP_INSTALL_FLAGS``
+    build arg.
+
 * 2.3.0
 
   * Airflow 2.3 ``Dockerfile`` is now better optimized for caching and "standalone" which means that you