Build Images #104
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| # | |
| --- | |
| name: "Build Images" | |
| on: # yamllint disable-line rule:truthy | |
| workflow_run: | |
| workflows: ["CI Build"] | |
| types: ['requested'] | |
| env: | |
| MOUNT_LOCAL_SOURCES: "false" | |
| MOUNT_FILES: "true" | |
| FORCE_ANSWER_TO_QUESTIONS: "yes" | |
| FORCE_PULL_IMAGES: "true" | |
| CHECK_IMAGE_FOR_REBUILD: "true" | |
| SKIP_CHECK_REMOTE_IMAGE: "true" | |
| DB_RESET: "true" | |
| VERBOSE: "true" | |
| UPGRADE_TO_LATEST_CONSTRAINTS: false | |
| USE_GITHUB_REGISTRY: "true" | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| GITHUB_USERNAME: ${{ github.actor }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest" | |
| GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false" | |
| BUILD_IMAGES: ${{ secrets.AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE != 'false' }} | |
| INSTALL_PROVIDERS_FROM_SOURCES: "true" | |
| jobs: | |
| cancel-workflow-runs: | |
| timeout-minutes: 10 | |
| name: "Cancel workflow runs" | |
| runs-on: ubuntu-latest | |
| outputs: | |
| sourceHeadRepo: ${{ steps.source-run-info.outputs.sourceHeadRepo }} | |
| sourceHeadBranch: ${{ steps.source-run-info.outputs.sourceHeadBranch }} | |
| sourceHeadSha: ${{ steps.source-run-info.outputs.sourceHeadSha }} | |
| mergeCommitSha: ${{ steps.source-run-info.outputs.mergeCommitSha }} | |
| targetCommitSha: ${{ steps.source-run-info.outputs.targetCommitSha }} | |
| pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }} | |
| pullRequestLabels: ${{ steps.source-run-info.outputs.pullRequestLabels }} | |
| sourceEvent: ${{ steps.source-run-info.outputs.sourceEvent }} | |
| cacheDirective: ${{ steps.cache-directive.outputs.docker-cache }} | |
| buildImages: ${{ steps.build-images.outputs.buildImages }} | |
| upgradeToLatestConstraints: ${{ steps.upgrade-constraints.outputs.upgradeToLatestConstraints }} | |
| steps: | |
| - name: "Get information about the original trigger of the run" | |
| uses: potiuk/get-workflow-origin@2ef0b065db6b688a2231f8a7f464df1aac254328 # v1_2 | |
| id: source-run-info | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| sourceRunId: ${{ github.event.workflow_run.id }} | |
| - name: "Cancel duplicated 'CI Build' runs" | |
| uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| cancelMode: allDuplicates | |
| sourceRunId: ${{ github.event.workflow_run.id }} | |
| - name: "Output BUILD_IMAGES" | |
| id: build-images | |
| run: | | |
| # Workaround - jobs cannot access env variable in "ifs" | |
| # https://github.community/t/how-to-set-and-access-a-workflow-variable/17335/16 | |
| echo "::set-output name=buildImages::${BUILD_IMAGES}" | |
| - name: "Cancel duplicated 'Build Image' runs" | |
| # We find duplicates of our own "Build Image" runs - due to a missing feature | |
| # in GitHub Actions, we have to use Job names to match Event/Repo/Branch matching | |
| # trick ¯\_(ツ)_/¯. We name the build-info job appropriately | |
| # and then we try to find and cancel all the jobs with the same Event + Repo + Branch as the | |
| # current Event/Repo/Branch combination. | |
| uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 | |
| with: | |
| cancelMode: namedJobs | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| notifyPRCancel: true | |
| jobNameRegexps: > | |
| [".*Event: ${{ steps.source-run-info.outputs.sourceEvent }} | |
| Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }} | |
| Branch: ${{ steps.source-run-info.outputs.sourceHeadBranch }}.*"] | |
| if: env.BUILD_IMAGES == 'true' | |
| - name: "Cancel all 'CI Build' runs where some jobs failed" | |
| # We find any of the "CI Build" workflow runs, where any of the important jobs | |
| # failed. The important jobs are selected by the regexp array below. | |
| # We also produce list of canceled "CI Build' runs as output, so that we | |
| # can cancel all the matching "Build Images" workflow runs in the two following steps. | |
| # Yeah. Adding to the complexity ¯\_(ツ)_/¯. | |
| uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 | |
| id: cancel-failed | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| cancelMode: failedJobs | |
| sourceRunId: ${{ github.event.workflow_run.id }} | |
| notifyPRCancel: true | |
| jobNameRegexps: > | |
| ["^Pylint$", "^Static checks", "^Build docs$", "^Spell check docs$", "^Backport packages$", | |
| "^Provider packages", "^Checks: Helm tests$", "^Test OpenAPI*"] | |
| - name: "Extract canceled failed runs" | |
| # We use this step to build regexp that will be used to match the Source Run id in | |
| # the build-info job below. If we cancelled some "CI Build" runs in the "cancel-failed' step | |
| # above - we want to cancel also the corresponding "Build Images" runs. Again we have | |
| # to match the jobs using job name rather than use proper API because that feature | |
| # is currently missing in GitHub Actions ¯\_(ツ)_/¯. | |
| id: extract-cancelled-failed-runs | |
| if: steps.cancel-failed.outputs.cancelledRuns != '[]' | |
| run: | | |
| REGEXP="Source Run id: " | |
| SEPARATOR="" | |
| for run_id in $(echo "${{ steps.cancel-failed.outputs.cancelledRuns }}" | jq '.[]') | |
| do | |
| REGEXP="${REGEXP}${SEPARATOR}(${run_id})" | |
| SEPARATOR="|" | |
| done | |
| echo "::set-output name=matching-regexp::[\"${REGEXP}\"]" | |
| - name: "Cancel triggered 'Build Images' runs for the cancelled failed runs" | |
| # In case we do have some cancelled jobs in the "cancel-failed" step above | |
| # We take the extracted regexp array prepared in the previous step and we use | |
| # it to cancel any jobs that have matching names containing Source Run Id: | |
| # followed by one of the run ids. Yes I know it's super complex ¯\_(ツ)_/¯. | |
| if: env.BUILD_IMAGES == 'true' && steps.cancel-failed.outputs.cancelledRuns != '[]' | |
| uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 | |
| with: | |
| cancelMode: namedJobs | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| notifyPRCancel: true | |
| jobNameRegexps: ${{ steps.extract-cancelled-failed-runs.outputs.matching-regexp }} | |
| - name: "Cancel duplicated 'CodeQL' runs" | |
| uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 | |
| id: cancel | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| cancelMode: allDuplicates | |
| workflowFileName: 'codeql-analysis.yml' | |
| - name: "Set Docker Cache Directive" | |
| id: cache-directive | |
| run: | | |
| if [[ ${{ steps.source-run-info.outputs.sourceEvent }} == 'schedule' ]]; then | |
| echo "::set-output name=docker-cache::disabled" | |
| else | |
| echo "::set-output name=docker-cache::pulled" | |
| fi | |
| - name: "Set upgrade to latest constraints" | |
| id: upgrade-constraints | |
| run: | | |
| if [[ ${{ steps.cancel.outputs.sourceEvent == 'push' || | |
| steps.cancel.outputs.sourceEvent == 'scheduled' }} == 'true' ]]; then | |
| echo "::set-output name=upgradeToLatestConstraints::${{ github.sha }}" | |
| else | |
| echo "::set-output name=upgradeToLatestConstraints::false" | |
| fi | |
| - name: "Cancel all duplicated 'Build Image' runs" | |
| # We find duplicates of all "Build Image" runs - due to a missing feature | |
| # in GitHub Actions, we have to use Job names to match Event/Repo/Branch matching | |
| # trick ¯\_(ツ)_/¯. We name the build-info job appropriately and then we try to match | |
| # all the jobs with the same Event + Repo + Branch match and cancel all the duplicates for those | |
| # This might cancel own run, so this is the last step in the job | |
| uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 | |
| with: | |
| cancelMode: allDuplicatedNamedJobs | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| notifyPRCancel: true | |
| selfPreservation: false | |
| jobNameRegexps: '["Event: \\S* Repo: \\S* Branch: \\S* "]' | |
| build-info: | |
| # The name is such long because we are using it to cancel duplicated 'Build Images' runs | |
| # by matching Event/Repo/Branch. This is a workaround for a missing feature of GitHub | |
| # Actions to link the source workflow run and the triggered workflow_run one. | |
| # We are also cancelling SourceRunId in case we determine that we should cancel the source | |
| # Run because of some failing jobs in the source run. Again ¯\_(ツ)_/¯. | |
| name: > | |
| Event: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }} | |
| Repo: ${{ needs.cancel-workflow-runs.outputs.sourceHeadRepo }} | |
| Branch: ${{ needs.cancel-workflow-runs.outputs.sourceHeadBranch }} | |
| Run id: ${{ github.run_id }} | |
| Source Run id: ${{ github.event.workflow_run.id }} | |
| Sha: ${{ github.sha }} | |
| Source Sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }} | |
| Merge commit Sha: ${{ needs.cancel-workflow-runs.outputs.mergeCommitSha }} | |
| Target commit Sha: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }} | |
| runs-on: ubuntu-latest | |
| needs: [cancel-workflow-runs] | |
| env: | |
| GITHUB_CONTEXT: ${{ toJson(github) }} | |
| outputs: | |
| pythonVersions: ${{ steps.selective-checks.python-versions }} | |
| allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }} | |
| defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} | |
| run-tests: ${{ steps.selective-checks.outputs.run-tests }} | |
| run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} | |
| image-build: ${{ steps.selective-checks.outputs.image-build }} | |
| if: > | |
| needs.cancel-workflow-runs.outputs.buildImages == 'true' | |
| steps: | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v2 | |
| - name: > | |
| Event: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }} | |
| Repo: ${{ needs.cancel-workflow-runs.outputs.sourceHeadRepo }} | |
| Branch: ${{ needs.cancel-workflow-runs.outputs.sourceHeadBranch }} | |
| Run id: ${{ github.run_id }} | |
| Source Run id: ${{ github.event.workflow_run.id }} | |
| Sha: ${{ github.sha }} | |
| Source Sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }} | |
| Merge commit Sha: ${{ needs.cancel-workflow-runs.outputs.mergeCommitSha }} | |
| Target commit Sha: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }} | |
| run: printenv | |
| - name: > | |
| Fetch incoming commit ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }} with its parent | |
| uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }} | |
| fetch-depth: 2 | |
| if: needs.cancel-workflow-runs.outputs.sourceEvent == 'pull_request' | |
| # checkout the master version again, to use the right script in master workflow | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v2 | |
| - name: Selective checks | |
| id: selective-checks | |
| env: | |
| EVENT_NAME: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }} | |
| INCOMING_COMMIT_SHA: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }} | |
| PR_LABELS: ${{ needs.cancel-workflow-runs.outputs.pullRequestLabels }} | |
| run: | | |
| if [[ ${EVENT_NAME} == "pull_request" ]]; then | |
| # Run selective checks | |
| ./scripts/ci/selective_ci_checks.sh "${INCOMING_COMMIT_SHA}" | |
| else | |
| # Run all checks | |
| ./scripts/ci/selective_ci_checks.sh | |
| fi | |
| build-images: | |
| timeout-minutes: 80 | |
| name: "Build ${{matrix.image-type}} images ${{matrix.python-version}}" | |
| runs-on: ubuntu-latest | |
| needs: [build-info, cancel-workflow-runs] | |
| strategy: | |
| matrix: | |
| # We need to attempt to build all possible versions here because workflow_run | |
| # event is run from master for both master and v1-10-tests | |
| python-version: ${{ fromJson(needs.build-info.outputs.allPythonVersions) }} | |
| image-type: [CI, PROD] | |
| fail-fast: true | |
| if: > | |
| needs.build-info.outputs.image-build == 'true' && | |
| needs.cancel-workflow-runs.outputs.buildImages == 'true' | |
| env: | |
| BACKEND: postgres | |
| PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} | |
| GITHUB_REGISTRY_PUSH_IMAGE_TAG: ${{ github.event.workflow_run.id }} | |
| UPGRADE_TO_LATEST_CONSTRAINTS: ${{ needs.cancel-workflow-runs.outputs.upgradeToLatestConstraints }} | |
| DOCKER_CACHE: ${{ needs.cancel-workflow-runs.outputs.cacheDirective }} | |
| steps: | |
| - name: > | |
| Checkout [${{ needs.cancel-workflow-runs.outputs.sourceEvent }}] | |
| Event: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }} | |
| Repo: ${{ needs.cancel-workflow-runs.outputs.sourceHeadRepo }} | |
| Branch: ${{ needs.cancel-workflow-runs.outputs.sourceHeadBranch }} | |
| Run id: ${{ github.run_id }} | |
| Source Run id: ${{ github.event.workflow_run.id }} | |
| Sha: ${{ github.sha }} | |
| Source Sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }} | |
| Merge commit Sha: ${{ needs.cancel-workflow-runs.outputs.mergeCommitSha }} | |
| Target commit Sha: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }} | |
| uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }} | |
| - name: "Retrieve DEFAULTS from the _initialization.sh" | |
| # We cannot "source" the script here because that would be a security problem (we cannot run | |
| # any code that comes from the sources coming from the PR. Therefore we extract the | |
| # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH via custom grep/awk/sed commands | |
| # Also 2.7 and 3.5 versions are not allowed to proceed on master | |
| id: defaults | |
| run: | | |
| DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \ | |
| awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') | |
| echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV | |
| DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \ | |
| scripts/ci/libraries/_initialization.sh | \ | |
| awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') | |
| echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV | |
| if [[ \ | |
| ${DEFAULT_BRANCH} != "master" || \ | |
| ( ${PYTHON_MAJOR_MINOR_VERSION} != "2.7" && ${PYTHON_MAJOR_MINOR_VERSION} != "3.5" ) \ | |
| ]]; then | |
| echo "::set-output name=proceed::true" | |
| else | |
| echo "::set-output name=proceed::false" | |
| fi | |
| - name: Initiate GitHub Checks for Building image | |
| uses: LouisBrunner/checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798 # v1.1.0 | |
| id: build-image-check | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| name: "Status of image build ${{ matrix.image-type }}: ${{ matrix.python-version }}" | |
| status: "in_progress" | |
| sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }} | |
| details_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| output: > | |
| {"summary": | |
| "Building the image: ${{ matrix.image-type }}: ${{ matrix.python-version }}. See the | |
| [Image Build](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) | |
| for details" } | |
| if: steps.defaults.outputs.proceed == 'true' | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} ) to 'main-airflow' to use main scripts" | |
| uses: actions/checkout@v2 | |
| with: | |
| path: "main-airflow" | |
| if: steps.defaults.outputs.proceed == 'true' | |
| - name: "Setup python" | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} | |
| if: steps.defaults.outputs.proceed == 'true' | |
| - name: "Override 'scripts/ci' with the ${{ github.ref }} version so that the PR cannot override it." | |
| # We should not override those scripts which become part of the image as they will not be | |
| # changed in the image built - we should only override those that are executed to build | |
| # the image. | |
| run: | | |
| rm -rf "scripts/ci" | |
| mv "main-airflow/scripts/ci" "scripts" | |
| if: steps.defaults.outputs.proceed == 'true' | |
| - name: "Free space" | |
| run: ./scripts/ci/tools/ci_free_space_on_ci.sh | |
| if: steps.defaults.outputs.proceed == 'true' | |
| - name: "Build CI images ${{ matrix.python-version }}:${{ github.event.workflow_run.id }}" | |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh | |
| if: matrix.image-type == 'CI' && steps.defaults.outputs.proceed == 'true' | |
| - name: "Push CI images ${{ matrix.python-version }}:${{ github.event.workflow_run.id }}" | |
| run: ./scripts/ci/images/ci_push_ci_images.sh | |
| if: matrix.image-type == 'CI' && steps.defaults.outputs.proceed == 'true' | |
| - name: "Build PROD images ${{ matrix.python-version }}:${{ github.event.workflow_run.id }}" | |
| run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh | |
| if: matrix.image-type == 'PROD' && steps.defaults.outputs.proceed == 'true' | |
| - name: "Push PROD images ${{ matrix.python-version }}:${{ github.event.workflow_run.id }}" | |
| run: ./scripts/ci/images/ci_push_production_images.sh | |
| if: matrix.image-type == 'PROD' && steps.defaults.outputs.proceed == 'true' | |
| - name: Update GitHub Checks for Building image with status | |
| uses: LouisBrunner/checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798 # v1.1.0 | |
| if: always() && steps.defaults.outputs.proceed == 'true' | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| check_id: ${{ steps.build-image-check.outputs.check_id }} | |
| status: "completed" | |
| sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }} | |
| conclusion: ${{ job.status }} | |
| details_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| output: > | |
| {"summary": | |
| "Building the image: ${{ matrix.image-type }}: ${{ matrix.python-version }}. See the | |
| [Image Build](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) | |
| for details" } | |
| cancel-on-build-cancel: | |
| name: "Cancel 'CI Build' jobs on build image cancelling." | |
| runs-on: ubuntu-latest | |
| if: cancelled() | |
| needs: [build-images] | |
| steps: | |
| - name: "Canceling the 'CI Build' source workflow in case of failure!" | |
| uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| cancelMode: self | |
| notifyPRCancel: true | |
| notifyPRCancelMessage: "Building image for the PR has been cancelled" | |
| sourceRunId: ${{ github.event.workflow_run.id }} | |
| cancel-on-build-failure: | |
| name: "Cancel 'CI Build' jobs on build image failing." | |
| runs-on: ubuntu-latest | |
| if: failure() | |
| needs: [build-images] | |
| steps: | |
| - name: "Canceling the 'CI Build' source workflow in case of failure!" | |
| uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| cancelMode: self | |
| notifyPRCancel: true | |
| notifyPRCancelMessage: | | |
| Building images for the PR has failed. Follow the the workflow link to check the reason. | |
| sourceRunId: ${{ github.event.workflow_run.id }} |