Optimizes image verification steps. (apache#14780)

So far we had matrix of builds that verified images - each
image was verified by separate matrix-based job and those
verifications were run after all images were alredy available.

This step optimizes it. Those steps are run in the same job
as "waiting for image", also they run in parallel which will
make them a bit faster.

This verification is fast and it can be run on any machine
in parallel without any problems.

.github/workflows/ci.yml

@@ -272,38 +272,9 @@ jobs:
         # The job will set the output "githubRegistry" - result of auto-detect which registry has
         # been used by checking where the image can be downloaded from.
         #
-        run: ./scripts/ci/images/ci_wait_for_all_ci_images.sh
+        run: ./scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh
 
 
-  verify-ci-images:
-    timeout-minutes: 20
-    name: "Verify CI Image Py${{matrix.python-version}}"
-    runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
-    needs: [build-info, ci-images]
-    strategy:
-      matrix:
-        python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
-    env:
-      RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
-      BACKEND: sqlite
-      GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
-    if: needs.build-info.outputs.image-build == 'true'
-    steps:
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v2
-        if: needs.build-info.outputs.waitForImage == 'true'
-        with:
-          persist-credentials: false
-      - name: "Free space"
-        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
-        if: |
-          needs.build-info.outputs.waitForImage == 'true'
-      - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
-        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
-      - name: "Verify CI image Py${{matrix.python-version}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
-        run: ./scripts/ci/images/ci_verify_ci_image.sh
-        if: needs.build-info.outputs.waitForImage == 'true'
-
   static-checks:
     timeout-minutes: 30
     name: "Static checks"
@@ -948,34 +919,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
         env:
           CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
             ${{needs.build-info.outputs.pythonVersionsListAsString}}
-        run: ./scripts/ci/images/ci_wait_for_all_prod_images.sh
-
-  verify-prod-images:
-    timeout-minutes: 20
-    name: "Verify Prod Image Py${{matrix.python-version}}"
-    runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
-    needs: [build-info, prod-images]
-    strategy:
-      matrix:
-        python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
-    env:
-      RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
-      BACKEND: sqlite
-      GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }}
-    steps:
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v2
-        if: needs.build-info.outputs.waitForImage == 'true'
-        with:
-          persist-credentials: false
-      - name: "Free space"
-        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
-        if: needs.build-info.outputs.waitForImage == 'true'
-      - name: "Prepare PROD Image"
-        run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
-      - name: "Verify PROD image Py${{matrix.python-version}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
-        run: ./scripts/ci/images/ci_verify_prod_image.sh
-        if: needs.build-info.outputs.waitForImage == 'true'
+        run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh
 
   tests-kubernetes:
     timeout-minutes: 50
@@ -1068,7 +1012,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       - prepare-provider-packages
       - test-provider-packages-released-airflow
       - prod-images
-      - verify-prod-images
       - docs
     if: >
       (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' ||
@@ -1116,7 +1059,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       - tests-mysql
       - tests-kubernetes
       - ci-images
-      - verify-ci-images
       - docs
     if: >
       (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' ||
@@ -1204,8 +1146,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
     needs:
       - build-info
       - constraints
-      - verify-ci-images
-      - verify-prod-images
+      - ci-images
+      - prod-images
       - static-checks
       - static-checks-pylint
       - tests-sqlite

scripts/ci/images/ci_wait_for_all_ci_images.sh → scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh

@@ -15,12 +15,27 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+set -euo pipefail
+
+# We cannot perform full initialization because it will be done later in the "single run" scripts
+# And some readonly variables are set there, therefore we only selectively reuse parallel lib needed
+LIBRARIES_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../libraries/" && pwd)
+# shellcheck source=scripts/ci/libraries/_all_libs.sh
+source "${LIBRARIES_DIR}/_all_libs.sh"
+
+initialization::set_output_color_variables
+
+parallel::make_sure_gnu_parallel_is_installed
+
 echo
 echo "Waiting for all CI images to appear: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}"
 echo
 
-for PYTHON_MAJOR_MINOR_VERSION in ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}
-do
-    export PYTHON_MAJOR_MINOR_VERSION
-    "$( dirname "${BASH_SOURCE[0]}" )/ci_wait_for_ci_image.sh"
-done
+parallel::initialize_monitoring
+
+parallel::monitor_progress
+
+# shellcheck disable=SC2086
+parallel --results "${PARALLEL_MONITORED_DIR}" \
+    "$( dirname "${BASH_SOURCE[0]}" )/ci_wait_for_and_verify_ci_image.sh" ::: \
+    ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}

scripts/ci/images/ci_wait_for_all_prod_images.sh → scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh

@@ -15,13 +15,27 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+set -euo pipefail
+
+# We cannot perform full initialization because it will be done later in the "single run" scripts
+# And some readonly variables are set there, therefore we only selectively reuse parallel lib needed
+LIBRARIES_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../libraries/" && pwd)
+# shellcheck source=scripts/ci/libraries/_all_libs.sh
+source "${LIBRARIES_DIR}/_all_libs.sh"
+
+initialization::set_output_color_variables
+
+parallel::make_sure_gnu_parallel_is_installed
 
 echo
 echo "Waiting for all PROD images to appear: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}"
 echo
 
-for PYTHON_MAJOR_MINOR_VERSION in ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}
-do
-    export PYTHON_MAJOR_MINOR_VERSION
-    "$( dirname "${BASH_SOURCE[0]}" )/ci_wait_for_prod_image.sh"
-done
+parallel::initialize_monitoring
+
+parallel::monitor_progress
+
+# shellcheck disable=SC2086
+parallel --results "${PARALLEL_MONITORED_DIR}" \
+    "$( dirname "${BASH_SOURCE[0]}" )/ci_wait_for_and_verify_prod_image.sh" ::: \
+    ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}

scripts/ci/images/ci_wait_for_ci_image.sh → scripts/ci/images/ci_wait_for_and_verify_ci_image.sh

@@ -15,9 +15,28 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+
+if [[ $1 == "" ]]; then
+  >&2 echo "Requires python MAJOR/MINOR version as first parameter"
+  exit 1
+fi
+
+export PYTHON_MAJOR_MINOR_VERSION=$1
+shift
+
+
 # shellcheck source=scripts/ci/libraries/_script_init.sh
 . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
 
+function pull_ci_image() {
+    local image_name_with_tag="${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+    start_end::group_start "Pulling ${image_name_with_tag} image"
+
+    push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_CI_IMAGE}" "${image_name_with_tag}"
+    start_end::group_end
+
+}
+
 push_pull_remove_images::check_if_github_registry_wait_for_image_enabled
 
 build_image::configure_docker_registry
@@ -29,4 +48,10 @@ start_end::group_start "Waiting for ${AIRFLOW_CI_IMAGE_NAME} image to appear"
 push_pull_remove_images::wait_for_github_registry_image \
     "${AIRFLOW_CI_IMAGE_NAME}${GITHUB_REGISTRY_IMAGE_SUFFIX}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
 
+build_images::prepare_ci_build
+
+pull_ci_image
+
+verify_image::verify_ci_image "${AIRFLOW_CI_IMAGE}"
+
 start_end::group_end

scripts/ci/images/ci_wait_for_prod_image.sh → scripts/ci/images/ci_wait_for_and_verify_prod_image.sh

@@ -15,17 +15,41 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+
+if [[ $1 == "" ]]; then
+  >&2 echo "Requires python MAJOR/MINOR version as first parameter"
+  exit 1
+fi
+
+export PYTHON_MAJOR_MINOR_VERSION=$1
+shift
+
+
 # shellcheck source=scripts/ci/libraries/_script_init.sh
 . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
 
+function pull_prod_image() {
+    local image_name_with_tag="${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+    start_end::group_start "Pulling the ${image_name_with_tag} image and tagging with ${AIRFLOW_PROD_IMAGE}"
+
+    push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PROD_IMAGE}" "${image_name_with_tag}"
+    start_end::group_end
+}
+
 push_pull_remove_images::check_if_github_registry_wait_for_image_enabled
 
 build_image::configure_docker_registry
 
 export AIRFLOW_PROD_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
+
 start_end::group_start "Waiting for ${AIRFLOW_PROD_IMAGE_NAME} image to appear"
 
 push_pull_remove_images::wait_for_github_registry_image \
     "${AIRFLOW_PROD_IMAGE_NAME}${GITHUB_REGISTRY_IMAGE_SUFFIX}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
-
 start_end::group_end
+
+build_images::prepare_prod_build
+
+pull_prod_image
+
+verify_image::verify_prod_image "${AIRFLOW_PROD_IMAGE}"

scripts/ci/libraries/_parallel.sh

@@ -71,6 +71,7 @@ function parallel::monitor_loop() {
         for directory in "${PARALLEL_MONITORED_DIR}"/*/*
         do
             parallel_process=$(basename "${directory}")
+
             echo "${COLOR_BLUE}### The last lines for ${parallel_process} process ###${COLOR_RESET}"
             echo
             tail -2 "${directory}/stdout" || true