Disable provider packages building for non-main branch builds (apache…

…#25056) * Disable provider packages building for non-main branch builds When running the builds from non-main, we do not want to use provider packages built locally, but we want to install those from PyPI. This is achieved by skipping the step of building providers in case default branch is not main. During this change it was also discovered that we do not need AIRFLOW_FROM_CONTEXT flag to indicate whether airflow is among those being installed from docker-context files. This is now detected automatically and airflow is installed from Pypi if not installed from docker context files. Providers and airflow are automatically detected and installed if they are present. Installing pre-cached dependencies from GitHub is skipped in case docker-context-files are used.
Jorricks · Jul 19, 2022 · 67a24ae · 67a24ae
1 parent b97d9b0
commit 67a24ae
Show file tree

Hide file tree

Showing 12 changed files with 208 additions and 250 deletions.
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
@@ -319,12 +319,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
           key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\
 ${{ hashFiles('.pre-commit-config.yaml') }}"
           restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}}
-        if: needs.build-info.outputs.default-branch == 'main'
       - name: "Regenerate dependencies in case they was modified manually so that we can build an image"
         run: >
           breeze static-checks --type update-providers-dependencies --all-files
           --show-diff-on-failure --color always || true
-        if: needs.build-info.outputs.default-branch == 'main'
       - name: >
           Pull CI image for PROD build:
           ${{ needs.build-info.outputs.default-python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
@@ -341,6 +339,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
           --package-list-file ./scripts/ci/installed_providers.txt
           --package-format wheel
           --version-suffix-for-pypi dev0
+        if: needs.build-info.outputs.default-branch == 'main'
       - name: "Prepare airflow package"
         run: breeze prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0
       - name: "Move dist packages to docker-context files"
@@ -354,8 +353,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
           --tag-as-latest
           --push-image
           --install-packages-from-context
-          --disable-airflow-repo-cache
-          --airflow-is-in-context
         env:
           UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
           DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -434,7 +434,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
           breeze prepare-provider-packages
           --package-list-file ./scripts/ci/installed_providers.txt
           --package-format wheel --version-suffix-for-pypi dev0
-        if: needs.build-info.outputs.in-workflow-build == 'true'
+        if: >
+          needs.build-info.outputs.in-workflow-build == 'true' &&
+          needs.build-info.outputs.default-branch == 'main'
       - name: "Prepare airflow package"
         run: breeze prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0
         if: needs.build-info.outputs.in-workflow-build == 'true'
@@ -450,8 +452,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
           --run-in-parallel
           --push-image
           --install-packages-from-context
-          --disable-airflow-repo-cache
-          --airflow-is-in-context
         env:
           UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
           DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
@@ -1720,17 +1720,18 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
           IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
       - name: "Cleanup dist and context file"
         run: rm -fv ./dist/* ./docker-context-files/*
+      - name: "Prepare airflow package for PROD build"
+        run: breeze prepare-airflow-package --package-format wheel
+        env:
+          VERSION_SUFFIX_FOR_PYPI: "dev0"
       - name: "Prepare providers packages for PROD build"
         run: >
           breeze prepare-provider-packages
           --package-list-file ./scripts/ci/installed_providers.txt
           --package-format wheel
         env:
           VERSION_SUFFIX_FOR_PYPI: "dev0"
-      - name: "Prepare airflow package for PROD build"
-        run: breeze prepare-airflow-package --package-format wheel
-        env:
-          VERSION_SUFFIX_FOR_PYPI: "dev0"
+        if: needs.build-info.outputs.default-branch == 'main'
       - name: "Start ARM instance"
         run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
         if: matrix.platform == 'linux/arm64'
@@ -1746,17 +1747,17 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
           PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
       - name: "Move dist packages to docker-context files"
         run: mv -v ./dist/*.whl ./docker-context-files
+        if: needs.build-info.outputs.default-branch == 'main'
       - name: "Push PROD cache ${{ matrix.python-version }} ${{ matrix.platform }}"
         run: >
           breeze build-prod-image
           --builder airflow_cache
-          --airflow-is-in-context
           --install-packages-from-context
           --prepare-buildx-cache
-          --disable-airflow-repo-cache
           --platform ${{ matrix.platform }}
         env:
           PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
+        if: needs.build-info.outputs.default-branch == 'main'
       - name: "Stop ARM instance"
         run: ./scripts/ci/images/ci_stop_arm_instance.sh
         if: always() && matrix.platform == 'linux/arm64'

diff --git a/Dockerfile b/Dockerfile
@@ -1283,16 +1283,23 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
 COPY --from=scripts common.sh install_pip_version.sh \
      install_airflow_dependencies_from_branch_tip.sh /scripts/docker/
 
+# We can set this value to true in case we want to install .whl/.tar.gz packages placed in the
+# docker-context-files folder. This can be done for both additional packages you want to install
+# as well as Airflow and Provider packages (it will be automatically detected if airflow
+# is installed from docker-context files rather than from PyPI)
+ARG INSTALL_PACKAGES_FROM_CONTEXT="false"
+
 # In case of Production build image segment we want to pre-install main version of airflow
 # dependencies from GitHub so that we do not have to always reinstall it from the scratch.
 # The Airflow (and providers in case INSTALL_PROVIDERS_FROM_SOURCES is "false")
 # are uninstalled, only dependencies remain
 # the cache is only used when "upgrade to newer dependencies" is not set to automatically
-# account for removed dependencies (we do not install them in the first place)
-# Upgrade to specific PIP version
+# account for removed dependencies (we do not install them in the first place) and in case
+# INSTALL_PACKAGES_FROM_CONTEXT is not set (because then caching it from main makes no sense).
 RUN bash /scripts/docker/install_pip_version.sh; \
     if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \
-          ${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \
+        ${INSTALL_PACKAGES_FROM_CONTEXT} == "false" && \
+        ${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \
         bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \
     fi
 
@@ -1302,8 +1309,7 @@ COPY --chown=airflow:0 ${AIRFLOW_SOURCES_WWW_FROM} ${AIRFLOW_SOURCES_WWW_TO}
 # hadolint ignore=SC2086, SC2010
 RUN if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then \
         # only prepare node modules and compile assets if the prod image is build from sources
-        # otherwise they are already compiled-in. We should do it in one step with removing artifacts \
-        # as we want to keep the final image small
+        # otherwise they are already compiled-in the package.
         bash /scripts/docker/prepare_node_modules.sh; \
         REMOVE_ARTIFACTS="true" BUILD_TYPE="prod" bash /scripts/docker/compile_www_assets.sh; \
         # Copy generated dist folder (otherwise it will be overridden by the COPY step below)
@@ -1319,22 +1325,14 @@ RUN if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then \
 
 # Add extra python dependencies
 ARG ADDITIONAL_PYTHON_DEPS=""
-# We can set this value to true in case we want to install .whl .tar.gz packages placed in the
-# docker-context-files folder. This can be done for both - additional packages you want to install
-# and for airflow as well (you have to set AIRFLOW_IS_IN_CONTEXT to true in this case)
-ARG INSTALL_PACKAGES_FROM_CONTEXT="false"
-# By default we install latest airflow from PyPI or sources. You can set this parameter to false
-# if Airflow is in the .whl or .tar.gz packages placed in `docker-context-files` folder and you want
-# to skip installing Airflow/Providers from PyPI or sources.
-ARG AIRFLOW_IS_IN_CONTEXT="false"
+
 # Those are additional constraints that are needed for some extras but we do not want to
 # Force them on the main Airflow package.
 # * dill<0.3.3 required by apache-beam
 ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="dill<0.3.3"
 
 ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} \
     INSTALL_PACKAGES_FROM_CONTEXT=${INSTALL_PACKAGES_FROM_CONTEXT} \
-    AIRFLOW_IS_IN_CONTEXT=${AIRFLOW_IS_IN_CONTEXT} \
     EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
 
 WORKDIR ${AIRFLOW_HOME}
@@ -1345,7 +1343,8 @@ COPY --from=scripts install_from_docker_context_files.sh install_airflow.sh \
 # hadolint ignore=SC2086, SC2010
 RUN if [[ ${INSTALL_PACKAGES_FROM_CONTEXT} == "true" ]]; then \
         bash /scripts/docker/install_from_docker_context_files.sh; \
-    elif [[ ${AIRFLOW_IS_IN_CONTEXT} == "false" ]]; then \
+    fi; \
+    if ! airflow version 2>/dev/null >/dev/null; then \
         bash /scripts/docker/install_airflow.sh; \
     fi; \
     if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \

diff --git a/IMAGES.rst b/IMAGES.rst
@@ -165,8 +165,7 @@ You can also skip installing airflow and install it from locally provided files
 
 .. code-block:: bash
 
-  breeze build-prod-image --python 3.7 --additional-extras=trino \
-     --airflow-is-in-context-pypi --install-packages-from-context
+  breeze build-prod-image --python 3.7 --additional-extras=trino --install-packages-from-context
 
 In this case you airflow and all packages (.whl files) should be placed in ``docker-context-files`` folder.
 

diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py
@@ -143,7 +143,6 @@
             "name": "Customization options (for specific customization needs)",
             "options": [
                 "--install-packages-from-context",
-                "--airflow-is-in-context",
                 "--cleanup-context",
                 "--disable-mysql-client-installation",
                 "--disable-mssql-client-installation",
@@ -266,14 +265,10 @@ def run_build_in_parallel(
     type=BetterChoice(ALLOWED_INSTALLATION_METHODS),
 )
 @option_install_providers_from_sources
-@click.option(
-    '--airflow-is-in-context',
-    help="If set Airflow is installed from docker-context-files only rather than from PyPI or sources.",
-    is_flag=True,
-)
 @click.option(
     '--install-packages-from-context',
-    help='Install wheels from local docker-context-files when building image.',
+    help='Install wheels from local docker-context-files when building image. '
+    'Implies --disable-airflow-repo-cache.',
     is_flag=True,
 )
 @click.option(

diff --git a/dev/breeze/src/airflow_breeze/params/build_prod_params.py b/dev/breeze/src/airflow_breeze/params/build_prod_params.py
@@ -43,7 +43,6 @@ class BuildProdParams(CommonBuildParams):
     airflow_constraints_mode: str = "constraints"
     default_constraints_branch: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
     airflow_constraints_reference: str = ""
-    airflow_is_in_context: bool = False
     cleanup_context: bool = False
     disable_airflow_repo_cache: bool = False
     disable_mssql_client_installation: bool = False
@@ -186,31 +185,19 @@ def extra_docker_build_flags(self) -> List[str]:
 
     @property
     def airflow_pre_cached_pip_packages(self) -> str:
-        airflow_pre_cached_pip = 'true'
-        if not self.airflow_is_in_context or self.disable_airflow_repo_cache:
-            airflow_pre_cached_pip = 'false'
-        return airflow_pre_cached_pip
+        return 'false' if self.disable_airflow_repo_cache else 'true'
 
     @property
     def install_mssql_client(self) -> str:
-        install_mssql = 'true'
-        if self.disable_mssql_client_installation:
-            install_mssql = 'false'
-        return install_mssql
+        return 'false' if self.disable_mssql_client_installation else 'true'
 
     @property
     def install_mysql_client(self) -> str:
-        install_mysql = 'true'
-        if self.disable_mysql_client_installation:
-            install_mysql = 'false'
-        return install_mysql
+        return 'false' if self.disable_mysql_client_installation else 'true'
 
     @property
     def install_postgres_client(self) -> str:
-        install_postgres = 'true'
-        if self.disable_postgres_client_installation:
-            install_postgres = 'false'
-        return install_postgres
+        return 'false' if self.disable_postgres_client_installation else 'true'
 
     @property
     def docker_context_files(self) -> str:
@@ -233,7 +220,6 @@ def required_image_args(self) -> List[str]:
             "airflow_image_date_created",
             "airflow_image_readme_url",
             "airflow_image_repository",
-            "airflow_is_in_context",
             "airflow_pre_cached_pip_packages",
             "airflow_version",
             "build_id",

diff --git a/dev/refresh_images.sh b/dev/refresh_images.sh
@@ -47,9 +47,7 @@ mv -v ./dist/*.whl ./docker-context-files
 breeze build-prod-image \
      --builder airflow_cache \
      --run-in-parallel \
-     --airflow-is-in-context \
      --install-packages-from-context \
      --prepare-buildx-cache \
-     --disable-airflow-repo-cache \
      --platform linux/amd64,linux/arm64 \
      --verbose
diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst
@@ -252,11 +252,6 @@ You can see some examples of those in:
 |                                    |                                          | .whl and .tar.gz files placed in the     |
 |                                    |                                          | ``docker-context-files``.                |
 +------------------------------------+------------------------------------------+------------------------------------------+
-| ``AIRFLOW_IS_IN_CONTEXT``          | ``false``                                | If set to true, it means that Airflow    |
-|                                    |                                          | and providers are available in context   |
-|                                    |                                          | and the image will not attempt to        |
-|                                    |                                          | install Airflow from PyPI or sources.    |
-+------------------------------------+------------------------------------------+------------------------------------------+
 
 Pre-caching PIP dependencies
 ............................

diff --git a/docs/docker-stack/changelog.rst b/docs/docker-stack/changelog.rst
@@ -73,8 +73,7 @@ Airflow 2.3
   * Add Multi-Platform support (AMD64/ARM64) in order to accommodate MacOS M1 users
   * Build parameters which control if packages and Airflow should be installed from context file were
     unified
-  * The ``INSTALL_FROM_PYPI`` arg was replaced to ``AIRFLOW_IS_IN_CONTEXT`` (with reverse meaning and
-    default changed to false)
+  * The ``INSTALL_FROM_PYPI`` arg was removed - it is automatically detected now.
   * The ``INSTALL_FROM_DOCKER_CONTEXT_FILES`` arg changed to ``INSTALL_PACKAGES_FROM_CONTEXT``
 
 Airflow 2.2