Production images on CI are now built from packages (apache#12685)

So far, the production images of Airflow were using sources
when they were built on CI. This PR changes that, to build
airflow + providers packages first and install them
rather than use sources as installation mechanism.

Part of apache#12261

.dockerignore

@@ -60,8 +60,9 @@
 !.github
 !empty
 
-# This folder is for you if you want to add any files to the docker context when you build your own
+# This folder is for you if you want to add any packages to the docker context when you build your own
 # docker image. most of other files and any new folder you add will be excluded by default
+# if you need other types of files - please add the extensions here.
 !docker-context-files
 
 # Avoid triggering context change on README change (new companies using Airflow)

.github/workflows/build-images-workflow-run.yml

@@ -23,7 +23,6 @@ on:  # yamllint disable-line rule:truthy
     types: ['requested']
 env:
   MOUNT_LOCAL_SOURCES: "false"
-  MOUNT_FILES: "true"
   FORCE_ANSWER_TO_QUESTIONS: "yes"
   FORCE_PULL_IMAGES: "true"
   CHECK_IMAGE_FOR_REBUILD: "true"
@@ -319,7 +318,7 @@ jobs:
       - name: "Setup python"
         uses: actions/setup-python@v2
         with:
-          python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+          python-version: ${{  matrix.python-version }}
         if: steps.defaults.outputs.proceed == 'true'
       - name: >
           Override "scripts/ci" with the "${{ needs.cancel-workflow-runs.outputs.targetBranch }}" branch
@@ -336,7 +335,8 @@ jobs:
         if: steps.defaults.outputs.proceed == 'true'
       - name: "Build CI images ${{ matrix.python-version }}:${{ github.event.workflow_run.id }}"
         run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
-        if: matrix.image-type == 'CI' && steps.defaults.outputs.proceed == 'true'
+        # locally built CI image is needed to prepare packages for PROD image build
+        if: steps.defaults.outputs.proceed == 'true'
       - name: "Push CI images ${{ matrix.python-version }}:${{ github.event.workflow_run.id }}"
         run: ./scripts/ci/images/ci_push_ci_images.sh
         if: matrix.image-type == 'CI' && steps.defaults.outputs.proceed == 'true'

.github/workflows/ci.yml

@@ -28,7 +28,6 @@ on:  # yamllint disable-line rule:truthy
 env:
 
   MOUNT_LOCAL_SOURCES: "false"
-  MOUNT_FILES: "true"
   FORCE_ANSWER_TO_QUESTIONS: "yes"
   FORCE_PULL_IMAGES: "true"
   CHECK_IMAGE_FOR_REBUILD: "true"
@@ -385,10 +384,16 @@ jobs:
         run: ./scripts/ci/provider_packages/ci_prepare_provider_readmes.sh
       - name: "Prepare provider packages"
         run: ./scripts/ci/provider_packages/ci_prepare_provider_packages.sh
-      - name: "Install and test provider packages via .whl files"
-        run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh whl
-      - name: "Install and test provider packages via .tar.gz files"
-        run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh tar.gz
+        env:
+          PACKAGE_FORMAT: "both"
+      - name: "Install and test provider packages via wheel files"
+        run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
+        env:
+          PACKAGE_FORMAT: "wheel"
+      - name: "Install and test provider packages via sdist files"
+        run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
+        env:
+          PACKAGE_FORMAT: "sdist"
       - name: "Upload provider package artifacts"
         uses: actions/upload-artifact@v2
         if: always()
@@ -414,6 +419,7 @@ jobs:
       PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
       VERSION_SUFFIX_FOR_PYPI: "a2"
       VERSION_SUFFIX_FOR_SVN: "a2"
+      PACKAGE_FORMAT: "both"
     if: needs.build-info.outputs.image-build == 'true'
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -430,12 +436,18 @@ jobs:
         run: ./scripts/ci/provider_packages/ci_prepare_provider_readmes.sh
       - name: "Prepare provider packages"
         run: ./scripts/ci/provider_packages/ci_prepare_provider_packages.sh
-      - name: "Prepare airflow packages so that they can be installed"
+        env:
+          PACKAGE_FORMAT: "both"
+      - name: "Prepare airflow package so that it can be installed"
         run: ./scripts/ci/build_airflow/ci_build_airflow_package.sh
-      - name: "Install and test provider packages via .whl files"
-        run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh whl
-      - name: "Install and test provider packages via .tar.gz files"
-        run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh tar.gz
+      - name: "Install and test provider packages via wheel files"
+        run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
+        env:
+          PACKAGE_FORMAT: "wheel"
+      - name: "Install and test provider packages via sdist files"
+        run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
+        env:
+          PACKAGE_FORMAT: "sdist"
       - name: "Upload provider package artifacts"
         uses: actions/upload-artifact@v2
         if: always()

.github/workflows/scheduled_quarantined.yml

@@ -24,7 +24,6 @@ on:  # yamllint disable-line rule:truthy
 
 env:
   MOUNT_LOCAL_SOURCES: "false"
-  MOUNT_FILES: "true"
   FORCE_ANSWER_TO_QUESTIONS: "yes"
   FORCE_PULL_IMAGES: "true"
   CHECK_IMAGE_FOR_REBUILD: "true"