diff --git a/.github/actions/build-ci-images/action.yml b/.github/actions/build-ci-images/action.yml index 81825f9a7915c..e93b2f4e71607 100644 --- a/.github/actions/build-ci-images/action.yml +++ b/.github/actions/build-ci-images/action.yml @@ -34,30 +34,6 @@ runs: - name: "Build & Push AMD64 CI images ${{ env.IMAGE_TAG }} ${{ env.PYTHON_VERSIONS }}" shell: bash run: breeze ci-image build --push --tag-as-latest --run-in-parallel --upgrade-on-failure - - name: "Generate source constraints" - shell: bash - run: > - breeze release-management generate-constraints --run-in-parallel - --airflow-constraints-mode constraints-source-providers -# - name: "Generate PyPI constraints" -# shell: bash -# run: > -# breeze release-management generate-constraints --run-in-parallel -# --airflow-constraints-mode constraints -# if: ${{ inputs.build-provider-packages != 'true' }} - - name: "Print dependency upgrade summary" - shell: bash - run: | - for PYTHON_VERSION in ${{ env.PYTHON_VERSIONS }}; do - echo "Summarizing Python $PYTHON_VERSION" - cat "files/constraints-${PYTHON_VERSION}/*.md" >> $GITHUB_STEP_SUMMARY || true - done - - name: "Upload constraint artifacts" - uses: actions/upload-artifact@v3 - with: - name: constraints - path: ./files/constraints-*/constraints-*.txt - retention-days: 7 - name: "Fix ownership" shell: bash run: breeze ci fix-ownership diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca44ad3199593..325865f741139 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -326,6 +326,61 @@ jobs: DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} BUILD_TIMEOUT_MINUTES: 70 + preview-constraints: + permissions: + contents: read + timeout-minutes: 20 + continue-on-error: true + name: > + Preview constraints + ${{needs.build-info.outputs.all-python-versions-list-as-string}} + runs-on: "${{needs.build-info.outputs.runs-on}}" + needs: [build-info, build-ci-images] + if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' + env: + RUNS_ON: "${{ needs.build-info.outputs.runs-on }}" + PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} + DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} + steps: + - name: Cleanup repo + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v3 + with: + persist-credentials: false + submodules: recursive + - name: "Install Breeze" + uses: ./.github/actions/breeze + - name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} + run: breeze ci-image pull --run-in-parallel --tag-as-latest + - name: "Source constraints" + shell: bash + run: > + breeze release-management generate-constraints --run-in-parallel + --airflow-constraints-mode constraints-source-providers || true + - name: "PyPI constraints" + shell: bash + timeout-minutes: 25 + run: > + breeze release-management generate-constraints --run-in-parallel + --airflow-constraints-mode constraints || true + - name: "Dependency upgrade summary" + shell: bash + run: | + for PYTHON_VERSION in ${{ env.PYTHON_VERSIONS }}; do + echo "Summarizing Python $PYTHON_VERSION" + cat "files/constraints-${PYTHON_VERSION}"/*.md >> $GITHUB_STEP_SUMMARY || true + done + - name: "Upload constraint artifacts" + uses: actions/upload-artifact@v3 + with: + name: constraints + path: ./files/constraints-*/constraints-*.txt + retention-days: 7 + - name: "Fix ownership" + run: breeze ci fix-ownership + if: always() + build-prod-images: timeout-minutes: 80 name: > @@ -1662,6 +1717,7 @@ jobs: env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} + PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' steps: - name: Cleanup repo @@ -1677,16 +1733,18 @@ jobs: run: breeze ci-image pull --run-in-parallel --tag-as-latest env: PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} - - name: "Generate constraints" + - name: "Generate source constraints" + run: | + breeze release-management generate-constraints \ + --run-in-parallel --airflow-constraints-mode constraints-source-providers + - name: "Generate no-providers constraints" run: | - breeze release-management generate-constraints --run-in-parallel \ - --airflow-constraints-mode constraints-source-providers breeze release-management generate-constraints \ --run-in-parallel --airflow-constraints-mode constraints-no-providers -# breeze release-management generate-constraints \ -# --run-in-parallel --airflow-constraints-mode constraints - env: - PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} + - name: "Generate PyPi constraints" + run: | + breeze release-management generate-constraints \ + --run-in-parallel --airflow-constraints-mode constraints - name: "Set constraints branch name" id: constraints-branch run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT} diff --git a/Dockerfile.ci b/Dockerfile.ci index 3e8fca1a58ce4..ec35aab734ed9 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1435,8 +1435,11 @@ RUN echo "Airflow version: ${AIRFLOW_VERSION}" # Those are additional constraints that are needed for some extras but we do not want to # force them on the main Airflow package. Currently we need no extra limits as PIP 23.1+ has much better # dependency resolution and we do not need to limit the versions of the dependencies -# aiobotocore is limited temporarily until it stops backtracking pip -ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="" +# +# Without grpcio-status limit, pip gets into very long backtracking +# We should attempt to remove it in the future +# +ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="grpcio-status>=1.55.0" ARG UPGRADE_TO_NEWER_DEPENDENCIES="false" ARG VERSION_SUFFIX_FOR_PYPI="" diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index c962856827abe..d1cf4d6a5b1ee 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -333,9 +333,10 @@ function install_all_providers_from_pypi_with_eager_upgrade() { # Installing it with Airflow makes sure that the version of package that matches current # Airflow requirements will be used. # shellcheck disable=SC2086 + set -x pip install ".[${NO_PROVIDERS_EXTRAS}]" "${packages_to_install[@]}" ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} \ --upgrade --upgrade-strategy eager - + set +x } function install_all_provider_packages_from_wheels() { diff --git a/scripts/in_container/run_generate_constraints.sh b/scripts/in_container/run_generate_constraints.sh index d3b4ad20e09cb..2c969c5db5bdd 100755 --- a/scripts/in_container/run_generate_constraints.sh +++ b/scripts/in_container/run_generate_constraints.sh @@ -152,5 +152,6 @@ echo echo "Constraints error markdown generated in ${CONSTRAINTS_MARKDOWN_DIFF}" echo +ls "${CONSTRAINTS_MARKDOWN_DIFF}" exit 0 diff --git a/setup.py b/setup.py index d3aee366f9343..cd1a56626dc20 100644 --- a/setup.py +++ b/setup.py @@ -1042,4 +1042,4 @@ def include_provider_namespace_packages_when_installing_from_sources() -> None: if __name__ == "__main__": - do_setup() # comment + do_setup() # comment to trigger upgrade to newer dependencies