From b91fd990a6b24cc63c9e50ead6dae1764a7416d5 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 2 Aug 2022 17:21:01 +0200 Subject: [PATCH] Move breeze commands to sub-commands (#25449) * mOve breeze commands to sub-commands Originally Python version of Breeze had only a handful of commands, while moving from Bash, but we are close to completion of the transition and the number of commands grew quite a lot bringit it on par to the commands we used to have in the Bash version of Breeze. However, only a small subset of the commands is actually useful to average developer, most of the more advanced commands are used in specific circumstances (release management, configuring Breeze once or rebuilding the image or even runing non-interactive test session are rarely used. Therefore it makes much more sense to surface the common commands as the top-level commands and move the less frequent commands to subcommands to move them out from the main help page. At the same time, the BREEZE.rst documentation got a little messy during the move and this is the right time to structure it similarly to breeze commands: * prerequisites and installation * first-time configuration * regular task description * more advanced tasks grupped in the same subcommands as in Breeze * diving deeper into details of Breeze implentation for those who wish to understand how Breeze works under-the-hood Aliases for the common commands that users could already get used to were created, and deprecation warnings are printed in casee those commands are used (guiding the user to the new commands to use). Less frequently used options for shell command are still available in `shell` but they have been removed from the default command to remove clutter. You can still used them by explicitly specifying the `shell` commnd. Configuration for rich click has been separated out from the command group implementation to separate packages. This allows for less problems with circular imports - none of the commands are needed when rich-click configuration is being prepared, which happens before main `click` command is parsed, which allows to run imports of the "code" as late as possible. * Update manage-dags-files.rst --- .github/workflows/build-images.yml | 36 +- .github/workflows/ci.yml | 193 +-- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/release_dockerhub_image.yml | 20 +- BREEZE.rst | 1353 +++++++++-------- CI.rst | 2 +- CONTRIBUTORS_QUICK_START.rst | 2 +- IMAGES.rst | 20 +- TESTING.rst | 20 +- breeze | 2 +- dev/MANUALLY_BUILDING_IMAGES.md | 4 +- dev/PROVIDER_PACKAGE_DETAILS.md | 18 +- dev/README_RELEASE_AIRFLOW.md | 10 +- dev/README_RELEASE_PROVIDER_PACKAGES.md | 12 +- dev/REFRESHING_CI_CACHE.md | 8 +- dev/TRACKING_BACKTRACKING_ISSUES.md | 8 +- ...11-unified-communication-with-the-users.md | 2 +- .../adr/0012-asking-user-for-confirmation.md | 2 +- dev/breeze/src/airflow_breeze/breeze.py | 15 + .../src/airflow_breeze/commands/__init__.py | 16 + .../airflow_breeze/commands/ci_commands.py | 60 +- .../commands/ci_commands_config.py | 62 + .../commands/ci_image_commands.py | 148 +- .../commands/ci_image_commands_config.py | 117 ++ .../commands/developer_commands.py | 156 +- .../commands/developer_commands_config.py | 161 ++ .../airflow_breeze/commands/main_command.py | 151 +- .../commands/production_image_commands.py | 164 +- .../production_image_commands_config.py | 132 ++ .../commands/release_management_commands.py | 110 +- .../release_management_commands_config.py | 100 ++ ...ntenance_commands.py => setup_commands.py} | 156 +- .../commands/setup_commands_config.py | 64 + .../commands/testing_commands.py | 48 +- .../commands/testing_commands_config.py | 57 + .../airflow_breeze/configure_rich_click.py | 53 +- .../params/common_build_params.py | 2 +- .../src/airflow_breeze/utils/click_utils.py | 21 + .../airflow_breeze/utils/common_options.py | 18 +- .../src/airflow_breeze/utils/console.py | 12 + .../utils/docker_command_utils.py | 4 +- dev/breeze/src/airflow_breeze/utils/image.py | 4 +- .../src/airflow_breeze/utils/path_utils.py | 4 +- .../src/airflow_breeze/utils/run_utils.py | 7 +- .../src/airflow_breeze/utils/visuals.py | 2 +- dev/refresh_images.sh | 10 +- docker_tests/docker_tests_utils.py | 4 +- images/breeze/output-build-docs.svg | 296 ---- images/breeze/output-build-image.svg | 332 ---- images/breeze/output-build-prod-image.svg | 396 ----- images/breeze/output-cleanup.svg | 120 -- images/breeze/output-command-hash-export.svg | 95 -- images/breeze/output-commands-hash.txt | 91 +- images/breeze/output-commands.svg | 384 ++--- images/breeze/output-compile-www-assets.svg | 115 -- images/breeze/output-config.svg | 144 -- images/breeze/output-docker-compose-tests.svg | 128 -- images/breeze/output-exec.svg | 99 -- .../breeze/output-find-newer-dependencies.svg | 132 -- images/breeze/output-fix-ownership.svg | 116 -- images/breeze/output-free-space.svg | 104 -- images/breeze/output-generate-constraints.svg | 180 --- .../breeze/output-prepare-airflow-package.svg | 124 -- .../output-prepare-provider-documentation.svg | 168 -- .../output-prepare-provider-packages.svg | 172 --- images/breeze/output-pull-image.svg | 174 --- images/breeze/output-pull-prod-image.svg | 174 --- .../output-regenerate-command-images.svg | 99 -- images/breeze/output-release-prod-images.svg | 158 -- images/breeze/output-resource-check.svg | 99 -- images/breeze/output-selective-check.svg | 144 -- images/breeze/output-self-upgrade.svg | 107 -- images/breeze/output-setup-autocomplete.svg | 116 -- images/breeze/output-shell.svg | 252 --- images/breeze/output-start-airflow.svg | 260 ---- images/breeze/output-static-checks.svg | 264 ---- images/breeze/output-stop.svg | 111 -- images/breeze/output-tests.svg | 192 --- images/breeze/output-verify-image.svg | 132 -- images/breeze/output-verify-prod-image.svg | 136 -- .../output-verify-provider-packages.svg | 172 --- images/breeze/output-version.svg | 95 -- images/breeze/output_build-docs.svg | 296 ++++ images/breeze/output_ci-image.svg | 111 ++ images/breeze/output_ci-image_build.svg | 328 ++++ images/breeze/output_ci-image_pull.svg | 162 ++ images/breeze/output_ci-image_verify.svg | 132 ++ images/breeze/output_ci.svg | 119 ++ .../output_ci_find-newer-dependencies.svg | 140 ++ images/breeze/output_ci_fix-ownership.svg | 116 ++ images/breeze/output_ci_free-space.svg | 104 ++ images/breeze/output_ci_resource-check.svg | 99 ++ images/breeze/output_ci_selective-check.svg | 144 ++ images/breeze/output_cleanup.svg | 120 ++ images/breeze/output_compile-www-assets.svg | 115 ++ images/breeze/output_exec.svg | 99 ++ images/breeze/output_prod-image.svg | 119 ++ images/breeze/output_prod-image_build.svg | 392 +++++ images/breeze/output_prod-image_pull.svg | 162 ++ images/breeze/output_prod-image_verify.svg | 136 ++ images/breeze/output_release-management.svg | 123 ++ ...elease-management_generate-constraints.svg | 180 +++ ...ase-management_prepare-airflow-package.svg | 124 ++ ...agement_prepare-provider-documentation.svg | 184 +++ ...e-management_prepare-provider-packages.svg | 184 +++ ...release-management_release-prod-images.svg | 158 ++ ...se-management_verify-provider-packages.svg | 172 +++ images/breeze/output_setup.svg | 127 ++ images/breeze/output_setup_autocomplete.svg | 116 ++ .../output_setup_command-hash-export.svg | 95 ++ images/breeze/output_setup_config.svg | 144 ++ ...output_setup_regenerate-command-images.svg | 99 ++ images/breeze/output_setup_self-upgrade.svg | 107 ++ images/breeze/output_setup_version.svg | 95 ++ images/breeze/output_shell.svg | 256 ++++ images/breeze/output_start-airflow.svg | 264 ++++ images/breeze/output_static-checks.svg | 264 ++++ images/breeze/output_stop.svg | 111 ++ images/breeze/output_testing.svg | 107 ++ .../output_testing_docker-compose-tests.svg | 128 ++ images/breeze/output_testing_tests.svg | 192 +++ .../pre_commit/pre_commit_breeze_cmd_line.py | 43 +- 122 files changed, 8255 insertions(+), 7365 deletions(-) create mode 100644 dev/breeze/src/airflow_breeze/commands/__init__.py create mode 100644 dev/breeze/src/airflow_breeze/commands/ci_commands_config.py create mode 100644 dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py create mode 100644 dev/breeze/src/airflow_breeze/commands/developer_commands_config.py create mode 100644 dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py create mode 100644 dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py rename dev/breeze/src/airflow_breeze/commands/{configuration_and_maintenance_commands.py => setup_commands.py} (75%) create mode 100644 dev/breeze/src/airflow_breeze/commands/setup_commands_config.py create mode 100644 dev/breeze/src/airflow_breeze/commands/testing_commands_config.py create mode 100644 dev/breeze/src/airflow_breeze/utils/click_utils.py delete mode 100644 images/breeze/output-build-docs.svg delete mode 100644 images/breeze/output-build-image.svg delete mode 100644 images/breeze/output-build-prod-image.svg delete mode 100644 images/breeze/output-cleanup.svg delete mode 100644 images/breeze/output-command-hash-export.svg delete mode 100644 images/breeze/output-compile-www-assets.svg delete mode 100644 images/breeze/output-config.svg delete mode 100644 images/breeze/output-docker-compose-tests.svg delete mode 100644 images/breeze/output-exec.svg delete mode 100644 images/breeze/output-find-newer-dependencies.svg delete mode 100644 images/breeze/output-fix-ownership.svg delete mode 100644 images/breeze/output-free-space.svg delete mode 100644 images/breeze/output-generate-constraints.svg delete mode 100644 images/breeze/output-prepare-airflow-package.svg delete mode 100644 images/breeze/output-prepare-provider-documentation.svg delete mode 100644 images/breeze/output-prepare-provider-packages.svg delete mode 100644 images/breeze/output-pull-image.svg delete mode 100644 images/breeze/output-pull-prod-image.svg delete mode 100644 images/breeze/output-regenerate-command-images.svg delete mode 100644 images/breeze/output-release-prod-images.svg delete mode 100644 images/breeze/output-resource-check.svg delete mode 100644 images/breeze/output-selective-check.svg delete mode 100644 images/breeze/output-self-upgrade.svg delete mode 100644 images/breeze/output-setup-autocomplete.svg delete mode 100644 images/breeze/output-shell.svg delete mode 100644 images/breeze/output-start-airflow.svg delete mode 100644 images/breeze/output-static-checks.svg delete mode 100644 images/breeze/output-stop.svg delete mode 100644 images/breeze/output-tests.svg delete mode 100644 images/breeze/output-verify-image.svg delete mode 100644 images/breeze/output-verify-prod-image.svg delete mode 100644 images/breeze/output-verify-provider-packages.svg delete mode 100644 images/breeze/output-version.svg create mode 100644 images/breeze/output_build-docs.svg create mode 100644 images/breeze/output_ci-image.svg create mode 100644 images/breeze/output_ci-image_build.svg create mode 100644 images/breeze/output_ci-image_pull.svg create mode 100644 images/breeze/output_ci-image_verify.svg create mode 100644 images/breeze/output_ci.svg create mode 100644 images/breeze/output_ci_find-newer-dependencies.svg create mode 100644 images/breeze/output_ci_fix-ownership.svg create mode 100644 images/breeze/output_ci_free-space.svg create mode 100644 images/breeze/output_ci_resource-check.svg create mode 100644 images/breeze/output_ci_selective-check.svg create mode 100644 images/breeze/output_cleanup.svg create mode 100644 images/breeze/output_compile-www-assets.svg create mode 100644 images/breeze/output_exec.svg create mode 100644 images/breeze/output_prod-image.svg create mode 100644 images/breeze/output_prod-image_build.svg create mode 100644 images/breeze/output_prod-image_pull.svg create mode 100644 images/breeze/output_prod-image_verify.svg create mode 100644 images/breeze/output_release-management.svg create mode 100644 images/breeze/output_release-management_generate-constraints.svg create mode 100644 images/breeze/output_release-management_prepare-airflow-package.svg create mode 100644 images/breeze/output_release-management_prepare-provider-documentation.svg create mode 100644 images/breeze/output_release-management_prepare-provider-packages.svg create mode 100644 images/breeze/output_release-management_release-prod-images.svg create mode 100644 images/breeze/output_release-management_verify-provider-packages.svg create mode 100644 images/breeze/output_setup.svg create mode 100644 images/breeze/output_setup_autocomplete.svg create mode 100644 images/breeze/output_setup_command-hash-export.svg create mode 100644 images/breeze/output_setup_config.svg create mode 100644 images/breeze/output_setup_regenerate-command-images.svg create mode 100644 images/breeze/output_setup_self-upgrade.svg create mode 100644 images/breeze/output_setup_version.svg create mode 100644 images/breeze/output_shell.svg create mode 100644 images/breeze/output_start-airflow.svg create mode 100644 images/breeze/output_static-checks.svg create mode 100644 images/breeze/output_stop.svg create mode 100644 images/breeze/output_testing.svg create mode 100644 images/breeze/output_testing_docker-compose-tests.svg create mode 100644 images/breeze/output_testing_tests.svg diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 65fc5dfc8a9d0..c3dcd2db1408f 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -157,7 +157,7 @@ jobs: env: PR_LABELS: "${{ steps.get-latest-pr-labels.outputs.pull-request-labels }}" COMMIT_REF: "${{ env.TARGET_COMMIT_SHA }}" - run: breeze selective-check + run: breeze ci selective-check - name: env run: printenv env: @@ -220,7 +220,7 @@ jobs: cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: Cache pre-commit envs uses: actions/cache@v3 with: @@ -238,7 +238,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: >- Build & Push AMD64 CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} ${{ needs.build-info.outputs.all-python-versions-list-as-string }} - run: breeze build-image --push-image --tag-as-latest --run-in-parallel + run: breeze ci-image build --push --tag-as-latest --run-in-parallel env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} @@ -246,16 +246,16 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }} - name: Push empty CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} if: failure() || cancelled() - run: breeze build-image --push-image --empty-image --run-in-parallel + run: breeze ci-image build --push --empty-image --run-in-parallel env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Candidates for pip resolver backtrack triggers" if: failure() || cancelled() run: > - breeze find-newer-dependencies --max-age 1 + breeze ci find-newer-dependencies --max-age 1 --python "${{ needs.build-info.outputs.default-python-version }}" - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() build-prod-images: @@ -313,7 +313,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: Cache pre-commit envs uses: actions/cache@v3 with: @@ -328,7 +328,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: > Pull CI image for PROD build: ${{ needs.build-info.outputs.default-python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - run: breeze pull-image --tag-as-latest + run: breeze ci-image pull --tag-as-latest env: # Always use default Python version of CI image for preparing packages PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} @@ -337,23 +337,25 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: rm -fv ./dist/* ./docker-context-files/* - name: "Prepare providers packages" run: > - breeze prepare-provider-packages + breeze release-management prepare-provider-packages --package-list-file ./scripts/ci/installed_providers.txt --package-format wheel --version-suffix-for-pypi dev0 if: needs.build-info.outputs.default-branch == 'main' - name: "Prepare airflow package" - run: breeze prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0 + run: > + breeze release-management prepare-airflow-package + --package-format wheel --version-suffix-for-pypi dev0 - name: "Move dist packages to docker-context files" run: mv -v ./dist/*.whl ./docker-context-files - name: >- Build & Push PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} ${{ needs.build-info.outputs.all-python-versions-list-as-string }} run: > - breeze build-prod-image + breeze prod-image build --run-in-parallel --tag-as-latest - --push-image + --push --install-packages-from-context env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} @@ -362,11 +364,11 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }} - name: Push empty PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} if: failure() || cancelled() - run: breeze build-prod-image --cleanup-context --push-image --empty-image --run-in-parallel + run: breeze prod-image build --cleanup-context --push --empty-image --run-in-parallel env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() @@ -420,14 +422,14 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" python-version: ${{ needs.build-info.outputs.default-python-version }} - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: "Start ARM instance" run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh - name: > Build ARM CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} ${{ needs.build-info.outputs.all-python-versions-list-as-string }} run: > - breeze build-image --run-in-parallel --builder airflow_cache --platform "linux/arm64" + breeze ci-image build --run-in-parallel --builder airflow_cache --platform "linux/arm64" env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} @@ -437,5 +439,5 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: ./scripts/ci/images/ci_stop_arm_instance.sh if: always() - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8c425143a475..4c7ca95fdf9b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -237,7 +237,7 @@ jobs: env: PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}" COMMIT_REF: "${{ github.sha }}" - run: breeze selective-check + run: breeze ci selective-check # Avoid having to specify the runs-on logic every time. We use the custom # env var AIRFLOW_SELF_HOSTED_RUNNER set only on our runners, but never # on the public runners @@ -330,7 +330,7 @@ jobs: - run: ./scripts/ci/install_breeze.sh if: needs.build-info.outputs.in-workflow-build == 'true' - name: "Free space" - run: breeze free-space + run: breeze ci free-space if: needs.build-info.outputs.in-workflow-build == 'true' - name: Cache pre-commit envs uses: actions/cache@v3 @@ -353,7 +353,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: > Build & Push CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} ${{ needs.build-info.outputs.all-python-versions-list-as-string }} - run: breeze build-image --push-image --tag-as-latest --run-in-parallel + run: breeze ci-image build --push --tag-as-latest --run-in-parallel env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} @@ -363,10 +363,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: "Candidates for pip resolver backtrack triggers" if: failure() || cancelled() run: > - breeze find-newer-dependencies --max-age 1 + breeze ci find-newer-dependencies --max-age 1 --python "${{ needs.build-info.outputs.default-python-version }}" - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() && needs.build-info.outputs.in-workflow-build == 'true' build-prod-images: @@ -404,7 +404,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - run: ./scripts/ci/install_breeze.sh if: needs.build-info.outputs.in-workflow-build == 'true' - name: "Free space" - run: breeze free-space + run: breeze ci free-space if: needs.build-info.outputs.in-workflow-build == 'true' - name: Cache pre-commit envs uses: actions/cache@v3 @@ -426,7 +426,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: > Pull CI image for PROD build: ${{ needs.build-info.outputs.default-python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}" - run: breeze pull-image --tag-as-latest + run: breeze ci-image pull --tag-as-latest env: # Always use default Python version of CI image for preparing packages PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} @@ -437,14 +437,16 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" if: needs.build-info.outputs.in-workflow-build == 'true' - name: "Prepare providers packages" run: > - breeze prepare-provider-packages + breeze release-management prepare-provider-packages --package-list-file ./scripts/ci/installed_providers.txt --package-format wheel --version-suffix-for-pypi dev0 if: > needs.build-info.outputs.in-workflow-build == 'true' && needs.build-info.outputs.default-branch == 'main' - name: "Prepare airflow package" - run: breeze prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0 + run: > + breeze release-management prepare-airflow-package + --package-format wheel --version-suffix-for-pypi dev0 if: needs.build-info.outputs.in-workflow-build == 'true' - name: "Move dist packages to docker-context files" run: mv -v ./dist/*.whl ./docker-context-files @@ -453,10 +455,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" Build & Push PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} ${{ needs.build-info.outputs.all-python-versions-list-as-string }} run: > - breeze build-prod-image + breeze prod-image build --tag-as-latest --run-in-parallel - --push-image + --push --install-packages-from-context env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} @@ -465,7 +467,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }} if: needs.build-info.outputs.in-workflow-build == 'true' - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() && needs.build-info.outputs.in-workflow-build == 'true' run-new-breeze-tests: @@ -486,7 +488,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: python -m pip install --editable ./dev/breeze/ - run: python -m pytest ./dev/breeze/ -n auto --color=yes - - run: breeze version + - run: breeze setup version tests-ui: timeout-minutes: 10 @@ -609,15 +611,15 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} id: wait-for-images - run: breeze pull-image --run-in-parallel --verify-image --wait-for-image --tag-as-latest + run: breeze ci-image pull --run-in-parallel --verify --wait-for-image --tag-as-latest env: PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() static-checks: @@ -651,10 +653,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}} - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: > Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - run: breeze pull-image --tag-as-latest + run: breeze ci-image pull --tag-as-latest env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Get Python version" @@ -668,7 +670,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" SKIP: ${{ needs.build-info.outputs.skip-pre-commits }} COLUMNS: "250" - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() # Those checks are run if no image needs to be built for checks. This is for simple changes that @@ -706,7 +708,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: "Get Python version" run: "echo \"::set-output name=host-python-version::$(python -c 'import platform; print(platform.python_version())')\"" @@ -721,7 +723,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" SKIP: ${{ needs.build-info.outputs.skip-pre-commits }} COLUMNS: "250" - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() docs: @@ -748,9 +750,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - run: breeze pull-image --tag-as-latest + run: breeze ci-image pull --tag-as-latest env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - uses: actions/cache@v3 @@ -778,7 +780,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" github.event_name == 'push' run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() prepare-test-provider-packages-wheel: @@ -806,24 +808,28 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: > Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - run: breeze pull-image --tag-as-latest + run: breeze ci-image pull --tag-as-latest env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Cleanup dist files" run: rm -fv ./dist/* - name: "Prepare provider documentation" - run: breeze prepare-provider-documentation --answer yes + run: breeze release-management prepare-provider-documentation --answer yes - name: "Prepare provider packages: wheel" - run: breeze prepare-provider-packages --package-format wheel --version-suffix-for-pypi dev0 + run: > + breeze release-management prepare-provider-packages + --package-format wheel --version-suffix-for-pypi dev0 - name: "Prepare airflow package: wheel" - run: breeze prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0 + run: > + breeze release-management prepare-airflow-package + --package-format wheel --version-suffix-for-pypi dev0 - name: "Install and test provider packages and airflow via wheel files" run: > - breeze verify-provider-packages --use-airflow-version wheel --use-packages-from-dist - --package-format wheel + breeze release-management verify-provider-packages + --use-airflow-version wheel --use-packages-from-dist --package-format wheel env: SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}" - name: "Remove airflow package and replace providers with 2.2-compliant versions" @@ -836,7 +842,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" apache-airflow-providers-celery==2.1.3 - name: "Install and test provider packages and airflow on Airflow 2.2 files" run: > - breeze verify-provider-packages --use-airflow-version 2.2.0 + breeze release-management verify-provider-packages --use-airflow-version 2.2.0 --use-packages-from-dist --package-format wheel --airflow-constraints-reference constraints-2.2.0 env: # The extras below are all extras that should be installed with Airflow 2.2.0 @@ -853,7 +859,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" sftp,singularity,slack,snowflake,sqlite,ssh,statsd,tableau,telegram,trino,vertica,\ virtualenv,yandex,zendesk" - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() prepare-test-provider-packages-sdist: @@ -881,18 +887,22 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: > Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - run: breeze pull-image --tag-as-latest + run: breeze ci-image pull --tag-as-latest env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Cleanup dist files" run: rm -fv ./dist/* - name: "Prepare provider packages: sdist" - run: breeze prepare-provider-packages --package-format sdist --version-suffix-for-pypi dev0 + run: > + breeze release-management prepare-provider-packages + --package-format sdist --version-suffix-for-pypi dev0 - name: "Prepare airflow package: sdist" - run: breeze prepare-airflow-package --package-format sdist --version-suffix-for-pypi dev0 + run: > + breeze release-management prepare-airflow-package + --package-format sdist --version-suffix-for-pypi dev0 - name: "Upload provider distribution artifacts" uses: actions/upload-artifact@v2 with: @@ -901,12 +911,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" retention-days: 1 - name: "Install and test provider packages and airflow via sdist files" run: > - breeze verify-provider-packages --use-airflow-version sdist --use-packages-from-dist - --package-format sdist + breeze release-management verify-provider-packages + --use-airflow-version sdist --use-packages-from-dist --package-format sdist env: SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}" - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() tests-helm: @@ -940,10 +950,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: > Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - run: breeze pull-image --tag-as-latest + run: breeze ci-image pull --tag-as-latest env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Tests: Helm" @@ -973,7 +983,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" path: "./files/coverage*.xml" retention-days: 7 - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() tests-postgres: @@ -1011,9 +1021,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - run: breeze pull-image --tag-as-latest + run: breeze ci-image pull --tag-as-latest env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Test downgrade" @@ -1047,7 +1057,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" path: "./files/coverage*.xml" retention-days: 7 - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() tests-mysql: @@ -1084,9 +1094,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - run: breeze pull-image --tag-as-latest + run: breeze ci-image pull --tag-as-latest env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Test downgrade" @@ -1120,7 +1130,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" path: "./files/coverage*.xml" retention-days: 7 - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() tests-mssql: @@ -1157,9 +1167,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - run: breeze pull-image --tag-as-latest + run: breeze ci-image pull --tag-as-latest env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Test downgrade" @@ -1191,7 +1201,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" path: "./files/coverage*.xml" retention-days: 7 - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() tests-sqlite: @@ -1226,9 +1236,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - run: breeze pull-image --tag-as-latest + run: breeze ci-image pull --tag-as-latest env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Test downgrade" @@ -1260,7 +1270,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" path: ./files/coverage*.xml retention-days: 7 - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() tests-quarantined: @@ -1303,9 +1313,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" echo "ISSUE_ID=10128" >> $GITHUB_ENV - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - run: breeze pull-image --tag-as-latest + run: breeze ci-image pull --tag-as-latest env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Tests: Quarantined" @@ -1341,7 +1351,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" path: "./files/coverage*.xml" retention-days: 7 - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() upload-coverage: @@ -1404,7 +1414,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: "Cache virtualenv environment" uses: actions/cache@v3 with: @@ -1415,12 +1425,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" # or from build-prod-images above. # We are utilising single job to wait for all images because this job merely waits # For the images to be available and test them. - run: breeze pull-prod-image --verify-image --wait-for-image --run-in-parallel + run: breeze prod-image pull --verify --wait-for-image --run-in-parallel env: PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() test-docker-compose-quick-start: @@ -1447,15 +1457,15 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: Pull PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - run: breeze pull-prod-image --tag-as-latest + run: breeze prod-image pull --tag-as-latest env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Test docker-compose quick start" - run: breeze docker-compose-tests + run: breeze testing docker-compose-tests - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() tests-kubernetes: @@ -1499,9 +1509,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - run: breeze pull-prod-image --run-in-parallel --tag-as-latest + run: breeze prod-image pull --run-in-parallel --tag-as-latest env: PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} @@ -1525,7 +1535,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" path: /tmp/kind_logs_* retention-days: 7 - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() tests-helm-executor-upgrade: @@ -1564,9 +1574,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - run: breeze pull-prod-image --run-in-parallel --tag-as-latest + run: breeze prod-image pull --run-in-parallel --tag-as-latest env: PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} @@ -1601,7 +1611,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" path: /tmp/kind_logs_* retention-days: 7 - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() constraints: @@ -1639,18 +1649,20 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - run: breeze pull-image --run-in-parallel --tag-as-latest + run: breeze ci-image pull --run-in-parallel --tag-as-latest env: PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Generate constraints" run: | - breeze generate-constraints --run-in-parallel \ + breeze release-management generate-constraints --run-in-parallel \ --airflow-constraints-mode constraints-source-providers - breeze generate-constraints --run-in-parallel --airflow-constraints-mode constraints-no-providers - breeze generate-constraints --run-in-parallel --airflow-constraints-mode constraints + breeze release-management generate-constraints \ + --run-in-parallel --airflow-constraints-mode constraints-no-providers + breeze release-management generate-constraints \ + --run-in-parallel --airflow-constraints-mode constraints env: PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} - name: "Set constraints branch name" @@ -1675,7 +1687,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" branch: ${{ steps.constraints-branch.outputs.branch }} directory: "repo" - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() # Push BuildX cache to GitHub Registry in Apache repository, if all tests are successful and build @@ -1714,11 +1726,11 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: > Pull CI image for PROD build ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}" - run: breeze pull-image --tag-as-latest + run: breeze ci-image pull --tag-as-latest env: # Always use default Python version of CI image for preparing packages PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} @@ -1726,12 +1738,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: "Cleanup dist and context file" run: rm -fv ./dist/* ./docker-context-files/* - name: "Prepare airflow package for PROD build" - run: breeze prepare-airflow-package --package-format wheel + run: breeze release-management prepare-airflow-package --package-format wheel env: VERSION_SUFFIX_FOR_PYPI: "dev0" - name: "Prepare providers packages for PROD build" run: > - breeze prepare-provider-packages + breeze release-management prepare-provider-packages --package-list-file ./scripts/ci/installed_providers.txt --package-format wheel env: @@ -1742,7 +1754,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" if: matrix.platform == 'linux/arm64' - name: "Push CI cache ${{ matrix.platform }}" run: > - breeze build-image + breeze ci-image build --builder airflow_cache --prepare-buildx-cache --run-in-parallel @@ -1750,14 +1762,15 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" --platform ${{ matrix.platform }} - name: "Push CI latest image ${{ matrix.platform }}" run: > - breeze build-image --tag-as-latest --push-image --run-in-parallel --platform ${{ matrix.platform }} + breeze ci-image build + --tag-as-latest --push --run-in-parallel --platform ${{ matrix.platform }} if: matrix.platform == 'linux/amd64' - name: "Move dist packages to docker-context files" run: mv -v ./dist/*.whl ./docker-context-files if: needs.build-info.outputs.default-branch == 'main' - name: "Push PROD cache ${{ matrix.python-version }} ${{ matrix.platform }}" run: > - breeze build-prod-image + breeze prod-image build --builder airflow_cache --install-packages-from-context --prepare-buildx-cache @@ -1765,14 +1778,14 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" if: needs.build-info.outputs.default-branch == 'main' - name: "Push PROD latest image ${{ matrix.platform }}" run: > - breeze build-prod-image --tag-as-latest --install-packages-from-context - --push-image --run-in-parallel --platform ${{ matrix.platform }} + breeze prod-image build --tag-as-latest --install-packages-from-context + --push --run-in-parallel --platform ${{ matrix.platform }} if: matrix.platform == 'linux/amd64' - name: "Stop ARM instance" run: ./scripts/ci/images/ci_stop_arm_instance.sh if: always() && matrix.platform == 'linux/arm64' - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() build-ci-arm-images: @@ -1813,14 +1826,14 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" python-version: ${{ needs.build-info.outputs.default-python-version }} - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: "Start ARM instance" run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh - name: > Build CI ARM images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} ${{ needs.build-info.outputs.all-python-versions-list-as-string }} run: > - breeze build-image + breeze ci-image build --run-in-parallel --builder airflow_cache --platform "linux/arm64" @@ -1833,5 +1846,5 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: ./scripts/ci/images/ci_stop_arm_instance.sh if: always() - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index eae0bd067b3b3..7e6e2d0f28441 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -55,7 +55,7 @@ jobs: id: selective-checks env: COMMIT_REF: "${{ github.sha }}" - run: breeze selective-check + run: breeze ci selective-check analyze: name: Analyze diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml index 18d2b585cd12f..17b8cb10312f6 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_dockerhub_image.yml @@ -61,7 +61,7 @@ jobs: - run: ./scripts/ci/install_breeze.sh - name: Selective checks id: selective-checks - run: breeze selective-check + run: breeze ci selective-check release-images: timeout-minutes: 120 name: "Release images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}" @@ -95,9 +95,9 @@ jobs: cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: breeze free-space + run: breeze ci free-space - name: Build CI image for PROD build ${{ needs.build-info.outputs.defaultPythonVersion }} - run: breeze build-image + run: breeze ci-image build env: PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }} - name: "Cleanup dist and context file" @@ -112,7 +112,7 @@ jobs: - name: > Release regular images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }} run: > - breeze release-prod-images + breeze release-management release-prod-images --dockerhub-repo ${{ github.repository }} --airflow-version ${{ github.event.inputs.airflowVersion }} ${{ needs.build-info.outputs.skipLatest }} @@ -121,7 +121,7 @@ jobs: - name: > Release slim images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }} run: > - breeze release-prod-images + breeze release-management release-prod-images --dockerhub-repo ${{ github.repository }} --airflow-version ${{ github.event.inputs.airflowVersion }} ${{ needs.build-info.outputs.skipLatest }} @@ -133,15 +133,15 @@ jobs: - name: > Verify regular AMD64 image: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }} run: > - breeze verify-prod-image - --pull-image + breeze prod-image verify + --pull --image-name ${{github.repository}}:${{github.event.inputs.airflowVersion}}-python${{matrix.python-version}} - name: > Verify slim AMD64 image: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }} run: > - breeze verify-prod-image - --pull-image + breeze prod-image verify + --pull --slim-image --image-name ${{github.repository}}:slim-${{github.event.inputs.airflowVersion}}-python${{matrix.python-version}} @@ -149,5 +149,5 @@ jobs: run: docker logout if: always() - name: "Fix ownership" - run: breeze fix-ownership + run: breeze ci fix-ownership if: always() diff --git a/BREEZE.rst b/BREEZE.rst index 94d770ca367db..c70590ced9358 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -207,7 +207,7 @@ periodically. For details see On WSL2 you might want to increase your Virtual Hard Disk by following: `Expanding the size of your WSL 2 Virtual Hard Disk `_ -There is a command ``breeze resource-check`` that you can run to check available resources. See below +There is a command ``breeze ci resource-check`` that you can run to check available resources. See below for details. Cleaning the environment @@ -254,19 +254,13 @@ Those are all available commands for Breeze and details about the commands are d Breeze installed this way is linked to your checked out sources of Airflow so Breeze will automatically use latest version of sources from ``./dev/breeze``. Sometimes, when dependencies are -updated ``breeze`` commands with offer you to ``self-upgrade`` (you just need to answer ``y`` when asked). +updated ``breeze`` commands with offer you to run self-upgrade (you just need to answer ``y`` when asked). You can always run such self-upgrade at any time: .. code-block:: bash - breeze self-upgrade - -Those are all available flags of ``self-upgrade`` command: - -.. image:: ./images/breeze/output-self-upgrade.svg - :width: 100% - :alt: Breeze self-upgrade + breeze setup self-upgrade If you have several checked out Airflow sources, Breeze will warn you if you are using it from a different source tree and will offer you to re-install from those sources - to make sure that you are using the right @@ -278,17 +272,11 @@ By default Breeze works on the version of Airflow that you run it in - in case y sources of Airflow and you installed Breeze from a directory - Breeze will be run on Airflow sources from where it was installed. -You can run ``breeze version`` command to see where breeze installed from and what are the current sources +You can run ``breeze setup version`` command to see where breeze installed from and what are the current sources that Breeze works on -Those are all available flags of ``version`` command: - -.. image:: ./images/breeze/output-version.svg - :width: 100% - :alt: Breeze version - Running Breeze for the first time -================================= +--------------------------------- The First time you run Breeze, it pulls and builds a local version of Docker images. It pulls the latest Airflow CI images from the @@ -308,25 +296,11 @@ You should set up the autocomplete option automatically by running: .. code-block:: bash - breeze setup-autocomplete - -You get the auto-completion working when you re-enter the shell (follow the instructions printed). -The command will warn you and not reinstall autocomplete if you already did, but you can -also force reinstalling the autocomplete via: - -.. code-block:: bash - - breeze setup-autocomplete --force + breeze setup autocomplete -Those are all available flags of ``setup-autocomplete`` command: - -.. image:: ./images/breeze/output-setup-autocomplete.svg - :width: 100% - :alt: Breeze setup autocomplete - -Customize your environment --------------------------- +Customizing your environment +---------------------------- When you enter the Breeze environment, automatically an environment file is sourced from ``files/airflow-breeze-config/variables.env``. @@ -369,27 +343,18 @@ inside container, to enable modified tmux configurations. -Running tests in the CI interactive environment -=============================================== - -Breeze helps with running tests in the same environment/way as CI tests are run. You can run various -types of tests while you enter Breeze CI interactive environment - this is described in detail -in ``_ +Regular development tasks +========================= -Here is the part of Breeze video which is relevant (note that it refers to the old ``./breeze-legacy`` -command and it is not yet available in the new ``breeze`` command): - -.. raw:: html +The regular Breeze development tasks are available as top-level commands. Those tasks are most often +used during the development, that's why they are available without any sub-command. More advanced +commands are separated to sub-commands. -
- - Airflow Breeze - Running tests - -
+Entering Breeze shell +--------------------- -Choosing different Breeze environment configuration -=================================================== +This is the most often used feature of breeze. It simply allows to enter the shell inside the Breeze +development environment (inside the Breeze container). You can use additional ``breeze`` flags to choose your environment. You can specify a Python version to use, and backend (the meta-data database). Thanks to that, with Breeze, you can recreate the same @@ -410,17 +375,6 @@ default settings. You can see which value of the parameters that can be stored persistently in cache marked with >VALUE< in the help of the commands. -Another part of configuration is enabling/disabling cheatsheet, asciiart. The cheatsheet and asciiart can -be disabled - they are "nice looking" and cheatsheet -contains useful information for first time users but eventually you might want to disable both if you -find it repetitive and annoying. - -With the config setting colour-blind-friendly communication for Breeze messages. By default we communicate -with the users about information/errors/warnings/successes via colour-coded messages, but we can switch -it off by passing ``--no-colour`` to config in which case the messages to the user printed by Breeze -will be printed using different schemes (italic/bold/underline) to indicate different kind of messages -rather than colours. - Here is the part of Breeze video which is relevant (note that it refers to the old ``./breeze-legacy`` command but it is very similar to current ``breeze`` command): @@ -433,275 +387,159 @@ command but it is very similar to current ``breeze`` command): -Those are all available flags of ``config`` command: - -.. image:: ./images/breeze/output-config.svg - :width: 100% - :alt: Breeze config - - -You can also dump hash of the configuration options used - this is mostly use to generate the dump -of help of the commands only when they change. - -.. image:: ./images/breeze/output-command-hash-export.svg - :width: 100% - :alt: Breeze command-hash-export - -Regenerating images for documentation -===================================== - -This documentation contains exported images with "help" of their commands and parameters. You can -regenerate all those images (which might be needed in case new version of rich is used) via -``regenerate-command-images`` command. - -.. image:: ./images/breeze/output-regenerate-command-images.svg - :width: 100% - :alt: Breeze regenerate-command-images +Building the documentation +-------------------------- +To build documentation in Breeze, use the ``build-docs`` command: -Compiling www assets -==================== +.. code-block:: bash -Airflow webserver needs to prepare www assets - compiled with node and yarn. The ``compile-www-assets`` -command takes care about it. This is needed when you want to run webserver inside of the breeze. + breeze build-docs -.. image:: ./images/breeze/output-compile-www-assets.svg - :width: 100% - :alt: Breeze compile-www-assets +Results of the build can be found in the ``docs/_build`` folder. -Starting complete Airflow installation -====================================== +The documentation build consists of three steps: -For testing Airflow you often want to start multiple components (in multiple terminals). Breeze has -built-in ``start-airflow`` command that start breeze container, launches multiple terminals using tmux -and launches all Airflow necessary components in those terminals. +* verifying consistency of indexes +* building documentation +* spell checking -When you are starting airflow from local sources, www asset compilation is automatically executed before. +You can choose only one stage of the two by providing ``--spellcheck-only`` or ``--docs-only`` after +extra ``--`` flag. .. code-block:: bash - breeze --python 3.7 --backend mysql start-airflow - + breeze build-docs --spellcheck-only -You can also use it to start any released version of Airflow from ``PyPI`` with the -``--use-airflow-version`` flag. +This process can take some time, so in order to make it shorter you can filter by package, using the flag +``--package-filter ``. The package name has to be one of the providers or ``apache-airflow``. For +instance, for using it with Amazon, the command would be: .. code-block:: bash - breeze --python 3.7 --backend mysql --use-airflow-version 2.2.5 start-airflow - -Those are all available flags of ``start-airflow`` command: - -.. image:: ./images/breeze/output-start-airflow.svg - :width: 100% - :alt: Breeze start-airflow - - -Troubleshooting -=============== - -If you are having problems with the Breeze environment, try the steps below. After each step you -can check whether your problem is fixed. - -1. If you are on macOS, check if you have enough disk space for Docker (Breeze will warn you if not). -2. Stop Breeze with ``breeze stop``. -3. Delete the ``.build`` directory and run ``breeze build-image``. -4. Clean up Docker images via ``breeze cleanup`` command. -5. Restart your Docker Engine and try again. -6. Restart your machine and try again. -7. Re-install Docker Desktop and try again. - -In case the problems are not solved, you can set the VERBOSE_COMMANDS variable to "true": + breeze build-docs --package-filter apache-airflow-providers-amazon -.. code-block:: +Often errors during documentation generation come from the docstrings of auto-api generated classes. +During the docs building auto-api generated files are stored in the ``docs/_api`` folder. This helps you +easily identify the location the problems with documentation originated from. - export VERBOSE_COMMANDS="true" +Those are all available flags of ``build-docs`` command: +.. image:: ./images/breeze/output_build-docs.svg + :width: 100% + :alt: Breeze build documentation -Then run the failed command, copy-and-paste the output from your terminal to the -`Airflow Slack `_ #airflow-breeze channel and -describe your problem. +Here is the part of Breeze video which is relevant (note that it refers to the old ``./breeze-legacy`` +command but it is very similar to current ``breeze`` command): -Uses of the Airflow Breeze environment -====================================== +.. raw:: html -Airflow Breeze is a bash script serving as a "swiss-army-knife" of Airflow testing. Under the -hood it uses other scripts that you can also run manually if you have problem with running the Breeze -environment. Breeze script allows performing the following tasks: +
+ + Airflow Breeze - Build docs + +
-Those are commands mostly used by contributors: +Running static checks +--------------------- -* Execute arbitrary command in the test environment with ``breeze shell`` command -* Enter interactive shell in CI container when ``shell`` (or no command) is specified -* Start containerised, development-friendly airflow installation with ``breeze start-airflow`` command -* Compile www assets for webserver ``breeze compile-www-assets`` command -* Build documentation with ``breeze build-docs`` command -* Initialize local virtualenv with ``./scripts/tools/initialize_virtualenv.py`` command -* Run static checks with autocomplete support ``breeze static-checks`` command -* Run test specified with ``breeze tests`` command -* Run docker-compose tests with ``breeze docker-compose-tests`` command. -* Build CI docker image with ``breeze build-image`` command -* Cleanup breeze with ``breeze cleanup`` command +You can run static checks via Breeze. You can also run them via pre-commit command but with auto-completion +Breeze makes it easier to run selective static checks. If you press after the static-check and if +you have auto-complete setup you should see auto-completable list of all checks available. -Additional management tasks: +.. code-block:: bash -* Join running interactive shell with ``breeze exec`` command -* Stop running interactive environment with ``breeze stop`` command -* Execute arbitrary docker-compose command with ``./breeze-legacy docker-compose`` command + breeze static-checks -t run-mypy -Tests ------ +The above will run mypy check for currently staged files. -You can regular unit tests with ``breeze`` in two different ways, either interactively run tests with -the default ``shell`` command or via the ``tests`` command. +You can also pass specific pre-commit flags for example ``--all-files`` : -Iterate on tests interactively -------------------------------- +.. code-block:: bash -You can simply enter the ``breeze`` container and run ``pytest`` command there. You can enter the -container via just ``breeze`` command or ``breeze shell`` command (the latter has more options -useful when you run integration or system tests). This is the best way if you want to interactively -run selected tests and iterate with the tests. Once you enter ``breeze`` environment it is ready -out-of-the-box to run your tests by running the right ``pytest`` command (autocomplete should help -you with autocompleting test name if you start typing ``pytest tests``). + breeze static-checks -t run-mypy --all-files -Here are few examples: +The above will run mypy check for all files. -Running single test: +There is a convenience ``--last-commit`` flag that you can use to run static check on last commit only: .. code-block:: bash - pytest tests/core/test_core.py::TestCore::test_check_operators + breeze static-checks -t run-mypy --last-commit -To run the whole test class: +The above will run mypy check for all files in the last commit. + +There is another convenience ``--commit-ref`` flag that you can use to run static check on specific commit: .. code-block:: bash - pytest tests/core/test_core.py::TestCore + breeze static-checks -t run-mypy --commit-ref 639483d998ecac64d0fef7c5aa4634414065f690 -You can re-run the tests interactively, add extra parameters to pytest and modify the files before -re-running the test to iterate over the tests. You can also add more flags when starting the -``breeze shell`` command when you run integration tests or system tests. Read more details about it -in the ``TESTING.rst `` where all the test types of our are explained and more information -on how to run them. +The above will run mypy check for all files in the 639483d998ecac64d0fef7c5aa4634414065f690 commit. +Any ``commit-ish`` reference from Git will work here (branch, tag, short/long hash etc.) -Running group of tests ------------------------ +If you ever need to get a list of the files that will be checked (for troubleshooting) use these commands: -You can also run tests via built-in ``breeze tests`` command - similarly as iterative ``pytest`` command -allows to run test individually, or by class or in any other way pytest allows to test them, but it -also allows to run the tests in the same test "types" that are used to run the tests in CI: Core, Always -API, Providers. This how our CI runs them - running each group in parallel to other groups and you can -replicate this behaviour. +.. code-block:: bash -Another interesting use of the ``breeze tests`` command is that you can easily specify sub-set of the -tests for Providers. ``breeze tests --test-type "Providers[airbyte,http]`` for example will only run -tests for airbyte and http providers. + breeze static-checks -t identity --verbose # currently staged files + breeze static-checks -t identity --verbose --from-ref $(git merge-base main HEAD) --to-ref HEAD # branch updates -Here is the detailed set of options for the ``breeze tests`` command. +Those are all available flags of ``static-checks`` command: -.. image:: ./images/breeze/output-tests.svg +.. image:: ./images/breeze/output_static-checks.svg :width: 100% - :alt: Breeze tests - -Kubernetes tests ----------------- - -* Manage KinD Kubernetes cluster and deploy Airflow to KinD cluster ``./breeze-legacy kind-cluster`` commands -* Run Kubernetes tests specified with ``./breeze-legacy kind-cluster tests`` command -* Enter the interactive kubernetes test environment with ``./breeze-legacy kind-cluster shell`` command - -CI Image tasks --------------- - -The image building is usually run for users automatically when needed, -but sometimes Breeze users might want to manually build, pull or verify the CI images. - -* Build CI docker image with ``breeze build-image`` command -* Pull CI images in parallel ``breeze pull-image`` command -* Verify CI image ``breeze verify-image`` command - -PROD Image tasks ----------------- - -Users can also build Production images when they are developing them. However when you want to -use the PROD image, the regular docker build commands are recommended. See -`building the image `_ - -* Build PROD image with ``breeze build-prod-image`` command -* Pull PROD image in parallel ``breeze pull-prod-image`` command -* Verify CI image ``breeze verify-prod-image`` command + :alt: Breeze static checks -Configuration and maintenance ------------------------------ +Here is the part of Breeze video which is relevant (note that it refers to the old ``./breeze-legacy`` +command but it is very similar to current ``breeze`` command): -* Cleanup breeze with ``breeze cleanup`` command -* Self-upgrade breeze with ``breeze self-upgrade`` command -* Setup autocomplete for Breeze with ``breeze setup-autocomplete`` command -* Print Breeze version with ``breeze version`` command -* Outputs hash of commands defined by ``breeze`` with ``command-hash-export`` (useful to avoid needless - regeneration of Breeze images) +.. raw:: html -CI tasks --------- -* Freeing space needed to run CI tests with ``breeze free-space`` command -* Fixing ownership of files in your repository with ``breeze fix-ownership`` command -* Checking available resources for docker with ``breeze resource-check`` command -* Deciding which tests should be run with ``breeze selective-check`` command +
+ + Airflow Breeze - Static checks + +
-Release tasks -------------- +.. note:: -Maintainers also can use Breeze for other purposes (those are commands that regular contributors likely -do not need or have no access to run). Those are usually connected with releasing Airflow: + When you run static checks, some of the artifacts (mypy_cache) is stored in docker-compose volume + so that it can speed up static checks execution significantly. However, sometimes, the cache might + get broken, in which case you should run ``breeze stop`` to clean up the cache. -* Prepare cache for CI: ``breeze build-image --prepare-build-cache`` and - ``breeze build-prod image --prepare-build-cache``(needs buildx plugin and write access to registry ghcr.io) -* Generate constraints with ``breeze generate-constraints`` (needed when conflicting changes are merged) -* Prepare airflow packages: ``breeze prepare-airflow-package`` (when releasing Airflow) -* Verify providers: ``breeze verify-provider-packages`` (when releasing provider packages) - including importing - the providers in an earlier airflow version. -* Prepare provider documentation ``breeze prepare-provider-documentation`` and prepare provider packages - ``breeze prepare-provider-packages`` (when releasing provider packages) -* Finding the updated dependencies since the last successful build when we have conflict with - ``breeze find-newer-dependencies`` command -* Release production images to DockerHub with ``breeze release-prod-images`` command +Starting Airflow +---------------- -Details of Breeze usage -======================= +For testing Airflow you often want to start multiple components (in multiple terminals). Breeze has +built-in ``start-airflow`` command that start breeze container, launches multiple terminals using tmux +and launches all Airflow necessary components in those terminals. -Database volumes in Breeze --------------------------- +When you are starting airflow from local sources, www asset compilation is automatically executed before. -Breeze keeps data for all it's integration in named docker volumes. Each backend and integration -keeps data in their own volume. Those volumes are persisted until ``breeze stop`` command. -You can also preserve the volumes by adding flag ``--preserve-volumes`` when you run the command. -Then, next time when you start Breeze, it will have the data pre-populated. +.. code-block:: bash -Those are all available flags of ``stop`` command: + breeze --python 3.7 --backend mysql start-airflow -.. image:: ./images/breeze/output-stop.svg - :width: 100% - :alt: Breeze stop -Image cleanup --------------- +You can also use it to start any released version of Airflow from ``PyPI`` with the +``--use-airflow-version`` flag. -Breeze uses docker images heavily and those images are rebuild periodically. This might cause extra -disk usage by the images. If you need to clean-up the images periodically you can run -``breeze cleanup`` command (by default it will skip removing your images before cleaning up but you -can also remove the images to clean-up everything by adding ``--all``). +.. code-block:: bash -Those are all available flags of ``cleanup`` command: + breeze --python 3.7 --backend mysql --use-airflow-version 2.2.5 start-airflow +Those are all available flags of ``start-airflow`` command: -.. image:: ./images/breeze/output-cleanup.svg +.. image:: ./images/breeze/output_start-airflow.svg :width: 100% - :alt: Breeze cleanup + :alt: Breeze start-airflow -Launching multiple terminals ----------------------------- +Launching multiple terminals in the same environment +---------------------------------------------------- Often if you want to run full airflow in the Breeze environment you need to launch multiple terminals and run ``airflow webserver``, ``airflow scheduler``, ``airflow worker`` in separate terminals. @@ -744,82 +582,70 @@ Here is the part of Breeze video which is relevant: Those are all available flags of ``exec`` command: -.. image:: ./images/breeze/output-exec.svg +.. image:: ./images/breeze/output_exec.svg :width: 100% :alt: Breeze exec -Additional tools ----------------- -To shrink the Docker image, not all tools are pre-installed in the Docker image. But we have made sure that there -is an easy process to install additional tools. +Compiling www assets +-------------------- -Additional tools are installed in ``/files/bin``. This path is added to ``$PATH``, so your shell will -automatically autocomplete files that are in that directory. You can also keep the binaries for your tools -in this directory if you need to. +Airflow webserver needs to prepare www assets - compiled with node and yarn. The ``compile-www-assets`` +command takes care about it. This is needed when you want to run webserver inside of the breeze. -**Installation scripts** +.. image:: ./images/breeze/output_compile-www-assets.svg + :width: 100% + :alt: Breeze compile-www-assets -For the development convenience, we have also provided installation scripts for commonly used tools. They are -installed to ``/files/opt/``, so they are preserved after restarting the Breeze environment. Each script -is also available in ``$PATH``, so just type ``install_`` to get a list of tools. +Breeze cleanup +-------------- -Currently available scripts: +Breeze uses docker images heavily and those images are rebuild periodically. This might cause extra +disk usage by the images. If you need to clean-up the images periodically you can run +``breeze setup cleanup`` command (by default it will skip removing your images before cleaning up but you +can also remove the images to clean-up everything by adding ``--all``). -* ``install_aws.sh`` - installs `the AWS CLI `__ including -* ``install_az.sh`` - installs `the Azure CLI `__ including -* ``install_gcloud.sh`` - installs `the Google Cloud SDK `__ including - ``gcloud``, ``gsutil``. -* ``install_imgcat.sh`` - installs `imgcat - Inline Images Protocol `__ - for iTerm2 (Mac OS only) -* ``install_java.sh`` - installs `the OpenJDK 8u41 `__ -* ``install_kubectl.sh`` - installs `the Kubernetes command-line tool, kubectl `__ -* ``install_snowsql.sh`` - installs `SnowSQL `__ -* ``install_terraform.sh`` - installs `Terraform `__ +Those are all available flags of ``cleanup`` command: -Launching Breeze integrations ------------------------------ -When Breeze starts, it can start additional integrations. Those are additional docker containers -that are started in the same docker-compose command. Those are required by some of the tests -as described in ``_. +.. image:: ./images/breeze/output_cleanup.svg + :width: 100% + :alt: Breeze setup cleanup -By default Breeze starts only airflow container without any integration enabled. If you selected -``postgres`` or ``mysql`` backend, the container for the selected backend is also started (but only the one -that is selected). You can start the additional integrations by passing ``--integration`` flag -with appropriate integration name when starting Breeze. You can specify several ``--integration`` flags -to start more than one integration at a time. -Finally you can specify ``--integration all`` to start all integrations. +Running arbitrary commands in container +--------------------------------------- -Once integration is started, it will continue to run until the environment is stopped with -``breeze stop`` command. or restarted via ``breeze restart`` command +More sophisticated usages of the breeze shell is using the ``breeze shell`` command - it has more parameters +and you can also use it to execute arbitrary commands inside the container. -Note that running integrations uses significant resources - CPU and memory. +.. code-block:: bash -Here is the part of Breeze video which is relevant (note that it refers to the old ``./breeze-legacy`` -command but it is very similar to current ``breeze`` command): + breeze shell "ls -la" -.. raw:: html +Those are all available flags of ``shell`` command: -
- - Airflow Breeze - Integrations - -
+.. image:: ./images/breeze/output_shell.svg + :width: 100% + :alt: Breeze shell -Managing CI images ------------------- -With Breeze you can build images that are used by Airflow CI and production ones. +Stopping the environment +------------------------ -For all development tasks, unit tests, integration tests, and static code checks, we use the -**CI image** maintained in GitHub Container Registry. +After starting up, the environment runs in the background and takes quite some memory which you might +want to free for other things you are running on your host. -The CI image is built automatically as needed, however it can be rebuilt manually with -``build-image`` command. The production -image should be built manually - but also a variant of this image is built automatically when -kubernetes tests are executed see `Running Kubernetes tests <#running-kubernetes-tests>`_ +You can always stop it via: + +.. code-block:: bash + + breeze stop + +Those are all available flags of ``stop`` command: + +.. image:: ./images/breeze/output_stop.svg + :width: 100% + :alt: Breeze stop Here is the part of Breeze video which is relevant (note that it refers to the old ``./breeze-legacy`` command but it is very similar to current ``breeze`` command): @@ -827,184 +653,270 @@ command but it is very similar to current ``breeze`` command): .. raw:: html -Building the image first time pulls a pre-built version of images from the Docker Hub, which may take some -time. But for subsequent source code changes, no wait time is expected. -However, changes to sensitive files like ``setup.py`` or ``Dockerfile.ci`` will trigger a rebuild -that may take more time though it is highly optimized to only rebuild what is needed. -Breeze has built in mechanism to check if your local image has not diverged too much from the -latest image build on CI. This might happen when for example latest patches have been released as new -Python images or when significant changes are made in the Dockerfile. In such cases, Breeze will -download the latest images before rebuilding because this is usually faster than rebuilding the image. +Troubleshooting +=============== -Those are all available flags of ``build-image`` command: +If you are having problems with the Breeze environment, try the steps below. After each step you +can check whether your problem is fixed. -.. image:: ./images/breeze/output-build-image.svg - :width: 100% - :alt: Breeze build-image +1. If you are on macOS, check if you have enough disk space for Docker (Breeze will warn you if not). +2. Stop Breeze with ``breeze stop``. +3. Delete the ``.build`` directory and run ``breeze ci-image build``. +4. Clean up Docker images via ``breeze cleanup`` command. +5. Restart your Docker Engine and try again. +6. Restart your machine and try again. +7. Re-install Docker Desktop and try again. -You can also pull the CI images locally in parallel with optional verification. +In case the problems are not solved, you can set the VERBOSE_COMMANDS variable to "true": + +.. code-block:: -Those are all available flags of ``pull-image`` command: + export VERBOSE_COMMANDS="true" -.. image:: ./images/breeze/output-pull-image.svg - :width: 100% - :alt: Breeze pull-image -Finally, you can verify CI image by running tests - either with the pulled/built images or -with an arbitrary image. +Then run the failed command, copy-and-paste the output from your terminal to the +`Airflow Slack `_ #airflow-breeze channel and +describe your problem. + +Advanced commands +================= + +Airflow Breeze is a bash script serving as a "swiss-army-knife" of Airflow testing. Under the +hood it uses other scripts that you can also run manually if you have problem with running the Breeze +environment. Breeze script allows performing the following tasks: + +Running tests +------------- + +You can run tests with ``breeze``. There are various tests type and breeze allows to run different test +types easily. You can run unit tests in different ways, either interactively run tests with the default +``shell`` command or via the ``testing`` commands. The latter allows to run more kinds of tests easily. -Those are all available flags of ``verify-image`` command: +Here is the detailed set of options for the ``breeze testing`` command. -.. image:: ./images/breeze/output-verify-image.svg +.. image:: ./images/breeze/output_testing.svg :width: 100% - :alt: Breeze verify-image + :alt: Breeze testing -Verifying providers -------------------- +Iterate on tests interactively via ``shell`` command +.................................................... -Breeze can also be used to verify if provider classes are importable and if they are following the -right naming conventions. This happens automatically on CI but you can also run it manually. +You can simply enter the ``breeze`` container and run ``pytest`` command there. You can enter the +container via just ``breeze`` command or ``breeze shell`` command (the latter has more options +useful when you run integration or system tests). This is the best way if you want to interactively +run selected tests and iterate with the tests. Once you enter ``breeze`` environment it is ready +out-of-the-box to run your tests by running the right ``pytest`` command (autocomplete should help +you with autocompleting test name if you start typing ``pytest tests``). + +Here are few examples: + +Running single test: .. code-block:: bash - breeze verify-provider-packages + pytest tests/core/test_core.py::TestCore::test_check_operators -You can also run the verification with an earlier airflow version to check for compatibility. +To run the whole test class: .. code-block:: bash - breeze verify-provider-packages --use-airflow-version 2.1.0 + pytest tests/core/test_core.py::TestCore -All the command parameters are here: +You can re-run the tests interactively, add extra parameters to pytest and modify the files before +re-running the test to iterate over the tests. You can also add more flags when starting the +``breeze shell`` command when you run integration tests or system tests. Read more details about it +in the ``TESTING.rst `` where all the test types of our are explained and more information +on how to run them. + +Here is the part of Breeze video which is relevant (note that it refers to the old ``./breeze-legacy`` +command and it is not yet available in the new ``breeze`` command): + +.. raw:: html -.. image:: ./images/breeze/output-verify-provider-packages.svg +
+ + Airflow Breeze - Running tests + +
+ + +Running unit/integration tests in groups +........................................ + +Another option you have is that you can also run tests via built-in ``breeze testing`` command. +The iterative ``pytest`` command allows to run test individually, or by class or in any other way +pytest allows to test them and run them interactively, but ``breeze testing`` command allows to +run the tests in the same test "types" that are used to run the tests in CI: for example Core, Always +API, Providers. This how our CI runs them - running each group in parallel to other groups and you can +replicate this behaviour. + +Another interesting use of the ``breeze testing tests`` command is that you can easily specify sub-set of the +tests for Providers. ``breeze testing tests --test-type "Providers[airbyte,http]`` for example will only run +tests for airbyte and http providers. + +Here is the detailed set of options for the ``breeze testing tests`` command. + +.. image:: ./images/breeze/output_testing_tests.svg :width: 100% - :alt: Breeze verify-provider-packages + :alt: Breeze testing tests -Preparing packages ------------------- +Running docker-compose tests +............................ -Breeze can also be used to prepare airflow packages - both "apache-airflow" main package and -provider packages. +You can use Breeze to run docker-compose tests. Those tests are run using Production image +and they are running test with the Quick-start docker compose we have. -You can read more about testing provider packages in -`TESTING.rst `_ +.. image:: ./images/breeze/output_testing_docker-compose-tests.svg + :width: 100% + :alt: Breeze testing docker-compose-tests -There are several commands that you can run in Breeze to manage and build packages: -* preparing Provider documentation files -* preparing Airflow packages -* preparing Provider packages +Running Kubernetes tests +........................ -Preparing provider documentation files is part of the release procedure by the release managers -and it is described in detail in `dev `_ . +Breeze helps with running Kubernetes tests in the same environment/way as CI tests are run. +Breeze helps to setup KinD cluster for testing, setting up virtualenv and downloads the right tools +automatically to run the tests. -The below example perform documentation preparation for provider packages. +You can: -.. code-block:: bash +* Manage KinD Kubernetes cluster and deploy Airflow to KinD cluster ``./breeze-legacy kind-cluster`` commands +* Run Kubernetes tests specified with ``./breeze-legacy kind-cluster tests`` command +* Enter the interactive kubernetes test environment with ``./breeze-legacy kind-cluster shell`` command - breeze prepare-provider-documentation +This is described in detail in `Testing Kubernetes `_. -By default, the documentation preparation runs package verification to check if all packages are -importable, but you can add ``--skip-package-verification`` to skip it. +Here is the part of Breeze video which is relevant (note that it refers to the old ``./breeze-legacy`` +command and it is not yet available in the current ``breeze`` command): -.. code-block:: bash +.. raw:: html - breeze prepare-provider-documentation --skip-package-verification +
+ + Airflow Breeze - Kubernetes tests + +
-You can also add ``--answer yes`` to perform non-interactive build. +CI Image tasks +-------------- -.. image:: ./images/breeze/output-prepare-provider-documentation.svg - :width: 100% - :alt: Breeze prepare-provider-documentation +The image building is usually run for users automatically when needed, +but sometimes Breeze users might want to manually build, pull or verify the CI images. -The packages are prepared in ``dist`` folder. Note, that this command cleans up the ``dist`` folder -before running, so you should run it before generating airflow package below as it will be removed. +.. image:: ./images/breeze/output_ci-image.svg + :width: 100% + :alt: Breeze ci-image -The below example builds provider packages in the wheel format. +For all development tasks, unit tests, integration tests, and static code checks, we use the +**CI image** maintained in GitHub Container Registry. -.. code-block:: bash +The CI image is built automatically as needed, however it can be rebuilt manually with +``ci image build`` command. - breeze prepare-provider-packages +Here is the part of Breeze video which is relevant (note that it refers to the old ``./breeze-legacy`` +command but it is very similar to current ``breeze`` command): -If you run this command without packages, you will prepare all packages, you can however specify -providers that you would like to build. By default ``both`` types of packages are prepared ( -``wheel`` and ``sdist``, but you can change it providing optional --package-format flag. +.. raw:: html -.. code-block:: bash +
+ + Airflow Breeze - Building images + +
- breeze prepare-provider-packages google amazon +Building the image first time pulls a pre-built version of images from the Docker Hub, which may take some +time. But for subsequent source code changes, no wait time is expected. +However, changes to sensitive files like ``setup.py`` or ``Dockerfile.ci`` will trigger a rebuild +that may take more time though it is highly optimized to only rebuild what is needed. -You can see all providers available by running this command: +Breeze has built in mechanism to check if your local image has not diverged too much from the +latest image build on CI. This might happen when for example latest patches have been released as new +Python images or when significant changes are made in the Dockerfile. In such cases, Breeze will +download the latest images before rebuilding because this is usually faster than rebuilding the image. -.. code-block:: bash +Building CI image +................. - breeze prepare-provider-packages --help +Those are all available flags of ``ci-image build`` command: -.. image:: ./images/breeze/output-prepare-provider-packages.svg +.. image:: ./images/breeze/output_ci-image_build.svg :width: 100% - :alt: Breeze prepare-provider-packages + :alt: Breeze ci-image build -You can prepare airflow packages using breeze: +Pulling CI image +................ -.. code-block:: bash +You can also pull the CI images locally in parallel with optional verification. - breeze prepare-airflow-package +Those are all available flags of ``pull`` command: -This prepares airflow .whl package in the dist folder. +.. image:: ./images/breeze/output_ci-image_pull.svg + :width: 100% + :alt: Breeze ci-image pull -Again, you can specify optional ``--package-format`` flag to build selected formats of airflow packages, -default is to build ``both`` type of packages ``sdist`` and ``wheel``. +Verifying CI image +.................. -.. code-block:: bash +Finally, you can verify CI image by running tests - either with the pulled/built images or +with an arbitrary image. - breeze prepare-airflow-package --package-format=wheel +Those are all available flags of ``verify`` command: -.. image:: ./images/breeze/output-prepare-airflow-package.svg +.. image:: ./images/breeze/output_ci-image_verify.svg :width: 100% - :alt: Breeze prepare-airflow-package + :alt: Breeze ci-image verify -Managing Production images --------------------------- +PROD Image tasks +---------------- + +Users can also build Production images when they are developing them. However when you want to +use the PROD image, the regular docker build commands are recommended. See +`building the image `_ + +.. image:: ./images/breeze/output_prod-image.svg + :width: 100% + :alt: Breeze prod-image The **Production image** is also maintained in GitHub Container Registry for Caching and in ``apache/airflow`` manually pushed for released versions. This Docker image (built using official Dockerfile) contains size-optimised Airflow installation with selected extras and dependencies. However in many cases you want to add your own custom version of the image - with added apt dependencies, -python dependencies, additional Airflow extras. Breeze's ``build-image`` command helps to build your own, +python dependencies, additional Airflow extras. Breeze's ``prod-image build`` command helps to build your own, customized variant of the image that contains everything you need. -You can switch to building the production image by using ``build-prod-image`` command. +You can building the production image manually by using ``prod-image build`` command. Note, that the images can also be built using ``docker build`` command by passing appropriate build-args as described in `IMAGES.rst `_ , but Breeze provides several flags that -makes it easier to do it. You can see all the flags by running ``breeze build-prod-image --help``, +makes it easier to do it. You can see all the flags by running ``breeze prod-image build --help``, but here typical examples are presented: .. code-block:: bash - breeze build-prod-image --additional-extras "jira" + breeze prod-image build --additional-extras "jira" This installs additional ``jira`` extra while installing airflow in the image. .. code-block:: bash - breeze build-prod-image --additional-python-deps "torchio==0.17.10" + breeze prod-image build --additional-python-deps "torchio==0.17.10" This install additional pypi dependency - torchio in specified version. - .. code-block:: bash - breeze build-prod-image --additional-dev-apt-deps "libasound2-dev" \ + breeze prod-image build --additional-dev-apt-deps "libasound2-dev" \ --additional-runtime-apt-deps "libasound2" This installs additional apt dependencies - ``libasound2-dev`` in the build image and ``libasound`` in the @@ -1017,16 +929,19 @@ suffix and they need to also be paired with corresponding runtime dependency add .. code-block:: bash - breeze build-prod-image --python 3.7 --additional-dev-deps "libasound2-dev" \ + breeze prod-image build --python 3.7 --additional-dev-deps "libasound2-dev" \ --additional-runtime-apt-deps "libasound2" Same as above but uses python 3.7. +Building PROD image +................... + Those are all available flags of ``build-prod-image`` command: -.. image:: ./images/breeze/output-build-prod-image.svg +.. image:: ./images/breeze/output_prod-image_build.svg :width: 100% - :alt: Breeze commands + :alt: Breeze prod-image build Here is the part of Breeze video which is relevant (note that it refers to the old ``./breeze-legacy`` command but it is very similar to current ``breeze`` command): @@ -1040,179 +955,342 @@ command but it is very similar to current ``breeze`` command): +Pulling PROD image +.................. + You can also pull PROD images in parallel with optional verification. Those are all available flags of ``pull-prod-image`` command: -.. image:: ./images/breeze/output-pull-prod-image.svg +.. image:: ./images/breeze/output_prod-image_pull.svg :width: 100% - :alt: Breeze pull-prod-image + :alt: Breeze prod-image pull + +Verifying PROD image +.................... Finally, you can verify PROD image by running tests - either with the pulled/built images or with an arbitrary image. Those are all available flags of ``verify-prod-image`` command: -.. image:: ./images/breeze/output-verify-prod-image.svg +.. image:: ./images/breeze/output_prod-image_verify.svg :width: 100% - :alt: Breeze verify-prod-image + :alt: Breeze prod-image verify -Releasing Production images to DockerHub ----------------------------------------- -The **Production image** can be released by release managers who have permissions to push the image. This -happens only when there is an RC candidate or final version of Airflow released. +Breeze setup +------------ -You release "regular" and "slim" images as separate steps. +Breeze has tools that you can use to configure defaults and breeze behaviours and perform some maintenance +operations that might be necessary when you add new commands in Breeze. It also allows to configure your +host operating system for Breeze autocompletion. -Releasing "regular" images: +Those are all available flags of ``setup`` command: -.. code-block:: bash +.. image:: ./images/breeze/output_setup.svg + :width: 100% + :alt: Breeze setup - breeze release-prod-images --airflow-version 2.4.0 +Breeze configuration +.................... -Or "slim" images: +You can configure and inspect settings of Breeze command via this command: Python version, Backend used as +well as backend versions. + +Another part of configuration is enabling/disabling cheatsheet, asciiart. The cheatsheet and asciiart can +be disabled - they are "nice looking" and cheatsheet +contains useful information for first time users but eventually you might want to disable both if you +find it repetitive and annoying. + +With the config setting colour-blind-friendly communication for Breeze messages. By default we communicate +with the users about information/errors/warnings/successes via colour-coded messages, but we can switch +it off by passing ``--no-colour`` to config in which case the messages to the user printed by Breeze +will be printed using different schemes (italic/bold/underline) to indicate different kind of messages +rather than colours. + +Those are all available flags of ``setup config`` command: + +.. image:: ./images/breeze/output_setup_config.svg + :width: 100% + :alt: Breeze setup config + +Setting up autocompletion +......................... + +You get the auto-completion working when you re-enter the shell (follow the instructions printed). +The command will warn you and not reinstall autocomplete if you already did, but you can +also force reinstalling the autocomplete via: .. code-block:: bash - breeze release-prod-images --airflow-version 2.4.0 --slim-images + breeze setup autocomplete --force -By default when you are releasing the "final" image, we also tag image with "latest" tags but this -step can be skipped if you pass the ``--skip-latest`` flag. +Those are all available flags of ``setup-autocomplete`` command: -These are all of the available flags for the ``release-prod-images`` command: +.. image:: ./images/breeze/output_setup_autocomplete.svg + :width: 100% + :alt: Breeze setup autocomplete + +Breeze version +.............. + +You can display Breeze version and with ``--verbose`` flag it can provide more information: where +Breeze is installed from and details about setup hashes. -.. image:: ./images/breeze/output-release-prod-images.svg +Those are all available flags of ``version`` command: + +.. image:: ./images/breeze/output_setup_version.svg :width: 100% - :alt: Release prod images + :alt: Breeze version -Running static checks ---------------------- +Breeze self-upgrade +................... -You can run static checks via Breeze. You can also run them via pre-commit command but with auto-completion -Breeze makes it easier to run selective static checks. If you press after the static-check and if -you have auto-complete setup you should see auto-completable list of all checks available. +You can self-upgrade breeze automatically. Those are all available flags of ``self-upgrade`` command: -.. code-block:: bash +.. image:: ./images/breeze/output_setup_self-upgrade.svg + :width: 100% + :alt: Breeze setup self-upgrade - breeze static-checks -t run-mypy +Exporting breeze command hash output +.................................... -The above will run mypy check for currently staged files. +You can also dump hash of the configuration options used - this is mostly used to generate the dump +of help of the commands only when they change by pre-commit.. -You can also pass specific pre-commit flags for example ``--all-files`` : +.. image:: ./images/breeze/output_setup_command-hash-export.svg + :width: 100% + :alt: Breeze config command-hash-export -.. code-block:: bash - breeze static-checks -t run-mypy --all-files +Regenerating images for documentation +..................................... -The above will run mypy check for all files. +This documentation contains exported images with "help" of their commands and parameters. You can +regenerate all those images (which might be needed in case new version of rich is used) via +``regenerate-command-images`` command. -There is a convenience ``--last-commit`` flag that you can use to run static check on last commit only: +.. image:: ./images/breeze/output_setup_regenerate-command-images.svg + :width: 100% + :alt: Breeze setup regenerate-command-images -.. code-block:: bash - breeze static-checks -t run-mypy --last-commit +CI tasks +-------- -The above will run mypy check for all files in the last commit. +Breeze hase a number of commands that are mostly used in CI environment to perform cleanup. -There is another convenience ``--commit-ref`` flag that you can use to run static check on specific commit: +.. image:: ./images/breeze/output_ci.svg + :width: 100% + :alt: Breeze ci commands -.. code-block:: bash +Running resource check +...................... - breeze static-checks -t run-mypy --commit-ref 639483d998ecac64d0fef7c5aa4634414065f690 +Breeze requires certain resources to be available - disk, memory, CPU. When you enter Breeze's shell, +the resources are checked and information if there is enough resources is displayed. However you can +manually run resource check any time by ``breeze ci resource-check`` command. -The above will run mypy check for all files in the 639483d998ecac64d0fef7c5aa4634414065f690 commit. -Any ``commit-ish`` reference from Git will work here (branch, tag, short/long hash etc.) +Those are all available flags of ``resource-check`` command: -If you ever need to get a list of the files that will be checked (for troubleshooting) use these commands: +.. image:: ./images/breeze/output_ci_resource-check.svg + :width: 100% + :alt: Breeze ci resource-check + +Freeing the space +................. + +When our CI runs a job, it needs all memory and disk it can have. We have a Breeze command that frees +the memory and disk space used. You can also use it clear space locally but it performs a few operations +that might be a bit invasive - such are removing swap file and complete pruning of docker disk space used. + +Those are all available flags of ``free-space`` command: + +.. image:: ./images/breeze/output_ci_free-space.svg + :width: 100% + :alt: Breeze ci free-space + +Fixing File/Directory Ownership +............................... + +On Linux, there is a problem with propagating ownership of created files (a known Docker problem). The +files and directories created in the container are not owned by the host user (but by the root user in our +case). This may prevent you from switching branches, for example, if files owned by the root user are +created within your sources. In case you are on a Linux host and have some files in your sources created +by the root user, you can fix the ownership of those files by running : + +.. code-block:: + + breeze ci fix-ownership + +Those are all available flags of ``fix-ownership`` command: + +.. image:: ./images/breeze/output_ci_fix-ownership.svg + :width: 100% + :alt: Breeze ci fix-ownership + +Selective check +............... + +When our CI runs a job, it needs to decide which tests to run, whether to build images and how much the test +should be run on multiple combinations of Python, Kubernetes, Backend versions. In order to optimize time +needed to run the CI Builds. You can also use the tool to test what tests will be run when you provide +a specific commit that Breeze should run the tests on. + +More details about the algorithm used to pick the right tests can be +found in `Selective Checks `_. + +Those are all available flags of ``selective-check`` command: + +.. image:: ./images/breeze/output_ci_selective-check.svg + :width: 100% + :alt: Breeze ci selective-check + +Tracking backtracking issues for CI builds +.......................................... + +When our CI runs a job, we automatically upgrade our dependencies in the ``main`` build. However, this might +lead to conflicts and ``pip`` backtracking for a long time (possibly forever) for dependency resolution. +Unfortunately those issues are difficult to diagnose so we had to invent our own tool to help us with +diagnosing them. This tool is ``find-newer-dependencies`` and it works in the way that it helps to guess +which new dependency might have caused the backtracking. The whole process is described in +`tracking backtracking issues `_. + +Those are all available flags of ``find-newer-dependencies`` command: + +.. image:: ./images/breeze/output_ci_find-newer-dependencies.svg + :width: 100% + :alt: Breeze ci find-newer-dependencies + +Release management tasks +------------------------ + +Maintainers also can use Breeze for other purposes (those are commands that regular contributors likely +do not need or have no access to run). Those are usually connected with releasing Airflow: + +.. image:: ./images/breeze/output_release-management.svg + :width: 100% + :alt: Breeze release management + +Breeze can be used to prepare airflow packages - both "apache-airflow" main package and +provider packages. + +Preparing provider documentation +................................ + +You can read more about testing provider packages in +`TESTING.rst `_ + +There are several commands that you can run in Breeze to manage and build packages: + +* preparing Provider documentation files +* preparing Airflow packages +* preparing Provider packages + +Preparing provider documentation files is part of the release procedure by the release managers +and it is described in detail in `dev `_ . + +The below example perform documentation preparation for provider packages. .. code-block:: bash - breeze static-checks -t identity --verbose # currently staged files - breeze static-checks -t identity --verbose --from-ref $(git merge-base main HEAD) --to-ref HEAD # branch updates + breeze release-management prepare-provider-documentation -Those are all available flags of ``static-checks`` command: +By default, the documentation preparation runs package verification to check if all packages are +importable, but you can add ``--skip-package-verification`` to skip it. + +.. code-block:: bash + + breeze release-management prepare-provider-documentation --skip-package-verification + +You can also add ``--answer yes`` to perform non-interactive build. -.. image:: ./images/breeze/output-static-checks.svg +.. image:: ./images/breeze/output_release-management_prepare-provider-documentation.svg :width: 100% - :alt: Breeze static checks + :alt: Breeze prepare-provider-documentation -Here is the part of Breeze video which is relevant (note that it refers to the old ``./breeze-legacy`` -command but it is very similar to current ``breeze`` command): +Preparing provider packages +........................... -.. raw:: html +You can use Breeze to prepare provider packages. -
- - Airflow Breeze - Static checks - -
+The packages are prepared in ``dist`` folder. Note, that this command cleans up the ``dist`` folder +before running, so you should run it before generating airflow package below as it will be removed. -.. note:: +The below example builds provider packages in the wheel format. - When you run static checks, some of the artifacts (mypy_cache) is stored in docker-compose volume - so that it can speed up static checks execution significantly. However, sometimes, the cache might - get broken, in which case you should run ``breeze stop`` to clean up the cache. +.. code-block:: bash + + breeze release-management prepare-provider-packages +If you run this command without packages, you will prepare all packages, you can however specify +providers that you would like to build. By default ``both`` types of packages are prepared ( +``wheel`` and ``sdist``, but you can change it providing optional --package-format flag. + +.. code-block:: bash + + breeze release-management prepare-provider-packages google amazon + +You can see all providers available by running this command: + +.. code-block:: bash + + breeze release-management prepare-provider-packages --help + +.. image:: ./images/breeze/output_release-management_prepare-provider-packages.svg + :width: 100% + :alt: Breeze prepare-provider-packages -Building the Documentation --------------------------- +Verifying provider packages +........................... -To build documentation in Breeze, use the ``build-docs`` command: +Breeze can also be used to verify if provider classes are importable and if they are following the +right naming conventions. This happens automatically on CI but you can also run it manually if you +just prepared provider packages and they are present in ``dist`` folder. .. code-block:: bash - breeze build-docs + breeze release-management verify-provider-packages -Results of the build can be found in the ``docs/_build`` folder. +You can also run the verification with an earlier airflow version to check for compatibility. -The documentation build consists of three steps: +.. code-block:: bash -* verifying consistency of indexes -* building documentation -* spell checking + breeze release-management verify-provider-packages --use-airflow-version 2.1.0 -You can choose only one stage of the two by providing ``--spellcheck-only`` or ``--docs-only`` after -extra ``--`` flag. +All the command parameters are here: -.. code-block:: bash +.. image:: ./images/breeze/output_release-management_verify-provider-packages.svg + :width: 100% + :alt: Breeze verify-provider-packages - breeze build-docs --spellcheck-only -This process can take some time, so in order to make it shorter you can filter by package, using the flag -``--package-filter ``. The package name has to be one of the providers or ``apache-airflow``. For -instance, for using it with Amazon, the command would be: +Preparing airflow packages +.......................... -.. code-block:: bash +You can prepare airflow packages using Breeze: - breeze build-docs --package-filter apache-airflow-providers-amazon +.. code-block:: bash -Often errors during documentation generation come from the docstrings of auto-api generated classes. -During the docs building auto-api generated files are stored in the ``docs/_api`` folder. This helps you -easily identify the location the problems with documentation originated from. + breeze release-management prepare-airflow-package -Those are all available flags of ``build-docs`` command: +This prepares airflow .whl package in the dist folder. -.. image:: ./images/breeze/output-build-docs.svg - :width: 100% - :alt: Breeze build documentation +Again, you can specify optional ``--package-format`` flag to build selected formats of airflow packages, +default is to build ``both`` type of packages ``sdist`` and ``wheel``. -Here is the part of Breeze video which is relevant (note that it refers to the old ``./breeze-legacy`` -command but it is very similar to current ``breeze`` command): +.. code-block:: bash -.. raw:: html + breeze release-management prepare-airflow-package --package-format=wheel -
- - Airflow Breeze - Build docs - -
+.. image:: ./images/breeze/output_release-management_prepare-airflow-package.svg + :width: 100% + :alt: Breeze release-management prepare-airflow-package Generating constraints ----------------------- +...................... Whenever setup.py gets modified, the CI main job will re-generate constraint files. Those constraint files are stored in separated orphan branches: ``constraints-main``, ``constraints-2-0``. @@ -1221,7 +1299,7 @@ Those are constraint files as described in detail in the ``_ contributing documentation. -You can use ``breeze generate-constraints`` command to manually generate constraints for +You can use ``breeze release-management generate-constraints`` command to manually generate constraints for all or selected python version and single constraint mode like this: .. warning:: @@ -1232,7 +1310,7 @@ all or selected python version and single constraint mode like this: .. code-block:: bash - breeze generate-constraints --airflow-constraints-mode constraints + breeze release-management generate-constraints --airflow-constraints-mode constraints Constraints are generated separately for each python version and there are separate constraints modes: @@ -1251,7 +1329,7 @@ Constraints are generated separately for each python version and there are separ Those are all available flags of ``generate-constraints`` command: -.. image:: ./images/breeze/output-generate-constraints.svg +.. image:: ./images/breeze/output_release-management_generate-constraints.svg :width: 100% :alt: Breeze generate-constraints @@ -1264,101 +1342,101 @@ This bumps the constraint files to latest versions and stores hash of setup.py. and setup.py hash files are stored in the ``files`` folder and while generating the constraints diff of changes vs the previous constraint files is printed. +Releasing Production images +........................... -Using local virtualenv environment in Your Host IDE ---------------------------------------------------- - -You can set up your host IDE (for example, IntelliJ's PyCharm/Idea) to work with Breeze -and benefit from all the features provided by your IDE, such as local and remote debugging, -language auto-completion, documentation support, etc. +The **Production image** can be released by release managers who have permissions to push the image. This +happens only when there is an RC candidate or final version of Airflow released. -To use your host IDE with Breeze: +You release "regular" and "slim" images as separate steps. -1. Create a local virtual environment: +Releasing "regular" images: - You can use any of the following wrappers to create and manage your virtual environments: - `pyenv `_, `pyenv-virtualenv `_, - or `virtualenvwrapper `_. +.. code-block:: bash -2. Use the right command to activate the virtualenv (``workon`` if you use virtualenvwrapper or - ``pyenv activate`` if you use pyenv. + breeze release-management release-prod-images --airflow-version 2.4.0 -3. Initialize the created local virtualenv: +Or "slim" images: .. code-block:: bash - ./scripts/tools/initialize_virtualenv.py - -.. warning:: - Make sure that you use the right Python version in this command - matching the Python version you have - in your local virtualenv. If you don't, you will get strange conflicts. + breeze release-management release-prod-images --airflow-version 2.4.0 --slim-images -4. Select the virtualenv you created as the project's default virtualenv in your IDE. +By default when you are releasing the "final" image, we also tag image with "latest" tags but this +step can be skipped if you pass the ``--skip-latest`` flag. -Note that you can also use the local virtualenv for Airflow development without Breeze. -This is a lightweight solution that has its own limitations. +These are all of the available flags for the ``release-prod-images`` command: -More details on using the local virtualenv are available in the `LOCAL_VIRTUALENV.rst `_. +.. image:: ./images/breeze/output_release-management_release-prod-images.svg + :width: 100% + :alt: Breeze release management release prod images -Here is the part of Breeze video which is relevant (note that it refers to the old ``./breeze-legacy`` -but it is not available in the ``breeze`` command): -.. raw:: html +Details of Breeze usage +======================= -
- - Airflow Breeze - Initialize virtualenv - -
+Database volumes in Breeze +-------------------------- -Running docker-compose tests ----------------------------- +Breeze keeps data for all it's integration in named docker volumes. Each backend and integration +keeps data in their own volume. Those volumes are persisted until ``breeze stop`` command. +You can also preserve the volumes by adding flag ``--preserve-volumes`` when you run the command. +Then, next time when you start Breeze, it will have the data pre-populated. -You can use Breeze to run docker-compose tests. Those tests are run using Production image -and they are running test with the Quick-start docker compose we have. +Those are all available flags of ``stop`` command: -.. image:: ./images/breeze/output-docker-compose-tests.svg +.. image:: ./images/breeze/output-stop.svg :width: 100% - :alt: Breeze generate-constraints + :alt: Breeze stop -Running Kubernetes tests ------------------------- +Additional tools +---------------- -Breeze helps with running Kubernetes tests in the same environment/way as CI tests are run. -Breeze helps to setup KinD cluster for testing, setting up virtualenv and downloads the right tools -automatically to run the tests. +To shrink the Docker image, not all tools are pre-installed in the Docker image. But we have made sure that there +is an easy process to install additional tools. -This is described in detail in `Testing Kubernetes `_. +Additional tools are installed in ``/files/bin``. This path is added to ``$PATH``, so your shell will +automatically autocomplete files that are in that directory. You can also keep the binaries for your tools +in this directory if you need to. -Here is the part of Breeze video which is relevant (note that it refers to the old ``./breeze-legacy`` -command and it is not yet available in the current ``breeze`` command): +**Installation scripts** -.. raw:: html +For the development convenience, we have also provided installation scripts for commonly used tools. They are +installed to ``/files/opt/``, so they are preserved after restarting the Breeze environment. Each script +is also available in ``$PATH``, so just type ``install_`` to get a list of tools. -
- - Airflow Breeze - Kubernetes tests - -
+Currently available scripts: -Stopping the interactive environment ------------------------------------- +* ``install_aws.sh`` - installs `the AWS CLI `__ including +* ``install_az.sh`` - installs `the Azure CLI `__ including +* ``install_gcloud.sh`` - installs `the Google Cloud SDK `__ including + ``gcloud``, ``gsutil``. +* ``install_imgcat.sh`` - installs `imgcat - Inline Images Protocol `__ + for iTerm2 (Mac OS only) +* ``install_java.sh`` - installs `the OpenJDK 8u41 `__ +* ``install_kubectl.sh`` - installs `the Kubernetes command-line tool, kubectl `__ +* ``install_snowsql.sh`` - installs `SnowSQL `__ +* ``install_terraform.sh`` - installs `Terraform `__ -After starting up, the environment runs in the background and takes precious memory. -You can always stop it via: +Launching Breeze integrations +----------------------------- -.. code-block:: bash +When Breeze starts, it can start additional integrations. Those are additional docker containers +that are started in the same docker-compose command. Those are required by some of the tests +as described in ``_. - breeze stop +By default Breeze starts only airflow container without any integration enabled. If you selected +``postgres`` or ``mysql`` backend, the container for the selected backend is also started (but only the one +that is selected). You can start the additional integrations by passing ``--integration`` flag +with appropriate integration name when starting Breeze. You can specify several ``--integration`` flags +to start more than one integration at a time. +Finally you can specify ``--integration all`` to start all integrations. -Those are all available flags of ``stop`` command: +Once integration is started, it will continue to run until the environment is stopped with +``breeze stop`` command. or restarted via ``breeze restart`` command -.. image:: ./images/breeze/output-stop.svg - :width: 100% - :alt: Breeze stop +Note that running integrations uses significant resources - CPU and memory. Here is the part of Breeze video which is relevant (note that it refers to the old ``./breeze-legacy`` command but it is very similar to current ``breeze`` command): @@ -1366,73 +1444,61 @@ command but it is very similar to current ``breeze`` command): .. raw:: html -Running resource check ----------------------- - -Breeze requires certain resources to be available - disk, memory, CPU. When you enter Breeze's shell, -the resources are checked and information if there is enough resources is displayed. However you can -manually run resource check any time by ``breeze resource-check`` command. -Those are all available flags of ``resource-check`` command: -.. image:: ./images/breeze/output-resource-check.svg - :width: 100% - :alt: Breeze resource-check +Using local virtualenv environment in Your Host IDE +--------------------------------------------------- +You can set up your host IDE (for example, IntelliJ's PyCharm/Idea) to work with Breeze +and benefit from all the features provided by your IDE, such as local and remote debugging, +language auto-completion, documentation support, etc. -Freeing the space ------------------ +To use your host IDE with Breeze: -When our CI runs a job, it needs all memory and disk it can have. We have a Breeze command that frees -the memory and disk space used. You can also use it clear space locally but it performs a few operations -that might be a bit invasive - such are removing swap file and complete pruning of docker disk space used. +1. Create a local virtual environment: -Those are all available flags of ``free-space`` command: + You can use any of the following wrappers to create and manage your virtual environments: + `pyenv `_, `pyenv-virtualenv `_, + or `virtualenvwrapper `_. -.. image:: ./images/breeze/output-free-space.svg - :width: 100% - :alt: Breeze free-space +2. Use the right command to activate the virtualenv (``workon`` if you use virtualenvwrapper or + ``pyenv activate`` if you use pyenv. +3. Initialize the created local virtualenv: -Selective check ---------------- +.. code-block:: bash -When our CI runs a job, it needs to decide which tests to run, whether to build images and how much the test -should be run on multiple combinations of Python, Kubernetes, Backend versions. In order to optimize time -needed to run the CI Builds. You can also use the tool to test what tests will be run when you provide -a specific commit that Breeze should run the tests on. + ./scripts/tools/initialize_virtualenv.py -More details about the algorithm used to pick the right tests can be -found in `Selective Checks `_. +.. warning:: + Make sure that you use the right Python version in this command - matching the Python version you have + in your local virtualenv. If you don't, you will get strange conflicts. -Those are all available flags of ``selective-check`` command: +4. Select the virtualenv you created as the project's default virtualenv in your IDE. -.. image:: ./images/breeze/output-selective-check.svg - :width: 100% - :alt: Breeze selective-check +Note that you can also use the local virtualenv for Airflow development without Breeze. +This is a lightweight solution that has its own limitations. +More details on using the local virtualenv are available in the `LOCAL_VIRTUALENV.rst `_. -Tracking backtracking issues for CI builds ------------------------------------------- +Here is the part of Breeze video which is relevant (note that it refers to the old ``./breeze-legacy`` +but it is not available in the ``breeze`` command): -When our CI runs a job, we automatically upgrade our dependencies in the ``main`` build. However, this might -lead to conflicts and ``pip`` backtracking for a long time (possibly forever) for dependency resolution. -Unfortunately those issues are difficult to diagnose so we had to invent our own tool to help us with -diagnosing them. This tool is ``find-newer-dependencies`` and it works in the way that it helps to guess -which new dependency might have caused the backtracking. The whole process is described in -`tracking backtracking issues `_. +.. raw:: html -Those are all available flags of ``find-newer-dependencies`` command: +
+ + Airflow Breeze - Initialize virtualenv + +
-.. image:: ./images/breeze/output-find-newer-dependencies.svg - :width: 100% - :alt: Breeze find-newer-dependencies Internal details of Breeze ========================== @@ -1474,22 +1540,6 @@ from your ``logs`` directory in the Airflow sources, so all logs created in the visible in the host as well. Every time you enter the container, the ``logs`` directory is cleaned so that logs do not accumulate. -Running Arbitrary commands in the Breeze environment ----------------------------------------------------- - -To run other commands/executables inside the Breeze Docker-based environment, use the -``breeze shell`` command. - -.. code-block:: bash - - breeze shell "ls -la" - -Those are all available flags of ``shell`` command: - -.. image:: ./images/breeze/output-shell.svg - :width: 100% - :alt: Breeze shell - Running "Docker Compose" commands --------------------------------- @@ -1516,25 +1566,6 @@ For automation scripts, you can export the ``ANSWER`` variable (and set it to export ANSWER="yes" -Fixing File/Directory Ownership -------------------------------- - -On Linux, there is a problem with propagating ownership of created files (a known Docker problem). The -files and directories created in the container are not owned by the host user (but by the root user in our -case). This may prevent you from switching branches, for example, if files owned by the root user are -created within your sources. In case you are on a Linux host and have some files in your sources created -by the root user, you can fix the ownership of those files by running : - -.. code-block:: - - breeze fix-ownership - -Those are all available flags of ``fix-ownership`` command: - -.. image:: ./images/breeze/output-fix-ownership.svg - :width: 100% - :alt: Breeze fix-ownership - Mounting Local Sources to Breeze -------------------------------- @@ -1625,7 +1656,7 @@ You can add dependencies to the ``Dockerfile.ci``, ``setup.py``. After you exit the container and re-run ``breeze``, Breeze detects changes in dependencies, asks you to confirm rebuilding the image and proceeds with rebuilding if you confirm (or skip it if you do not confirm). After rebuilding is done, Breeze drops you to shell. You may also use the -``build-image`` command to only rebuild CI image and not to go into shell. +``build`` command to only rebuild CI image and not to go into shell. Incremental apt Dependencies in the Dockerfile.ci during development .................................................................... diff --git a/CI.rst b/CI.rst index c058598449299..362dabcbf7969 100644 --- a/CI.rst +++ b/CI.rst @@ -693,7 +693,7 @@ In order to add a new version the following operations should be done (example u .. code-block:: bash - breeze build-image --python 3.10 + breeze ci-image build --python 3.10 * Find the 2 new images (prod, ci) created in `GitHub Container registry `_ diff --git a/CONTRIBUTORS_QUICK_START.rst b/CONTRIBUTORS_QUICK_START.rst index 452a72ad3bd97..2526d7b87f22f 100644 --- a/CONTRIBUTORS_QUICK_START.rst +++ b/CONTRIBUTORS_QUICK_START.rst @@ -218,7 +218,7 @@ Setting up Breeze .. code-block:: bash - $ breeze setup-autocomplete + $ breeze setup autocomplete 4. Initialize breeze environment with required python version and backend. This may take a while for first time. diff --git a/IMAGES.rst b/IMAGES.rst index 36e6f73229cc8..6883c64a479c1 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -85,13 +85,13 @@ You can build the CI image using current sources this command: .. code-block:: bash - breeze build-image + breeze ci-image build You can build the PROD image using current sources with this command: .. code-block:: bash - breeze build-prod-image + breeze prod-image build By adding ``--python `` parameter you can build the image version for the chosen Python version. @@ -104,13 +104,13 @@ For example if you want to build Python 3.7 version of production image with .. code-block:: bash - breeze build-prod-image --python 3.7 --extras "all" + breeze prod-image build --python 3.7 --extras "all" If you just want to add new extras you can add them like that: .. code-block:: bash - breeze build-prod-image --python 3.7 --additional-extras "all" + breeze prod-image build --python 3.7 --additional-extras "all" The command that builds the CI image is optimized to minimize the time needed to rebuild the image when the source code of Airflow evolves. This means that if you already have the image locally downloaded and @@ -128,7 +128,7 @@ parameter to Breeze: .. code-block:: bash - breeze build-prod-image --python 3.7 --additional-extras=trino --install-airflow-version=2.0.0 + breeze prod-image build --python 3.7 --additional-extras=trino --install-airflow-version=2.0.0 This will build the image using command similar to: @@ -165,7 +165,7 @@ You can also skip installing airflow and install it from locally provided files .. code-block:: bash - breeze build-prod-image --python 3.7 --additional-extras=trino --install-packages-from-context + breeze prod-image build --python 3.7 --additional-extras=trino --install-packages-from-context In this case you airflow and all packages (.whl files) should be placed in ``docker-context-files`` folder. @@ -192,21 +192,21 @@ or ``disabled`` flags when you run Breeze commands. For example: .. code-block:: bash - breeze build-image --python 3.7 --docker-cache local + breeze ci-image build --python 3.7 --docker-cache local Will build the CI image using local build cache (note that it will take quite a long time the first time you run it). .. code-block:: bash - breeze build-prod-image --python 3.7 --docker-cache registry + breeze prod-image build --python 3.7 --docker-cache registry Will build the production image with cache used from registry. .. code-block:: bash - breeze build-prod-image --python 3.7 --docker-cache disabled + breeze prod-image build --python 3.7 --docker-cache disabled Will build the production image from the scratch. @@ -327,7 +327,7 @@ the same image can be built using ``breeze`` (it supports auto-completion of the .. code-block:: bash - breeze build-prod-image --python 3.7 --additional-extras=jdbc --additional-python-deps="pandas" \ + breeze prod-image build --python 3.7 --additional-extras=jdbc --additional-python-deps="pandas" \ --additional-dev-apt-deps="gcc g++" --additional-runtime-apt-deps="default-jre-headless" You can customize more aspects of the image - such as additional commands executed before apt dependencies diff --git a/TESTING.rst b/TESTING.rst index ace3b7b6ceedb..37d23c16f89fe 100644 --- a/TESTING.rst +++ b/TESTING.rst @@ -169,37 +169,37 @@ to breeze. .. code-block:: bash - breeze tests tests/providers/http/hooks/test_http.py tests/core/test_core.py --db-reset --log-cli-level=DEBUG + breeze testing tests tests/providers/http/hooks/test_http.py tests/core/test_core.py --db-reset --log-cli-level=DEBUG You can run the whole test suite without adding the test target: .. code-block:: bash - breeze tests --db-reset + breeze testing tests --db-reset You can also specify individual tests or a group of tests: .. code-block:: bash - breeze tests --db-reset tests/core/test_core.py::TestCore + breeze testing tests --db-reset tests/core/test_core.py::TestCore You can also limit the tests to execute to specific group of tests .. code-block:: bash - breeze tests --test-type Core + breeze testing tests --test-type Core In case of Providers tests, you can run tests for all providers .. code-block:: bash - breeze tests --test-type Providers + breeze testing tests --test-type Providers You can also limit the set of providers you would like to run tests of .. code-block:: bash - breeze tests --test-type "Providers[airbyte,http]" + breeze testing tests --test-type "Providers[airbyte,http]" You can also write tests in "limited progress" mode (useful in the future to run CI). In this mode each @@ -208,7 +208,7 @@ after it completes. .. code-block:: bash - breeze tests --test-type Core --limit-progress-output + breeze testing tests --test-type Core --limit-progress-output Running Tests of a specified type from the Host @@ -585,7 +585,7 @@ need to run the following steps: .. code-block:: bash - breeze prepare-provider-packages [PACKAGE ...] + breeze release-management prepare-provider-packages [PACKAGE ...] If you run this command without packages, you will prepare all packages. However, You can specify providers that you would like to build if you just want to build few provider packages. @@ -596,7 +596,7 @@ before running, so you should run it before generating ``apache-airflow`` packag .. code-block:: bash - breeze prepare-airflow-package + breeze release-management prepare-airflow-package This prepares airflow .whl package in the dist folder. @@ -1111,7 +1111,7 @@ example, the below command will build google, postgres and mysql wheel packages: .. code-block:: bash - breeze prepare-provider-packages google postgres mysql + breeze release-management prepare-provider-packages google postgres mysql Those packages will be prepared in ./dist folder. This folder is mapped to /dist folder when you enter Breeze, so it is easy to automate installing those packages for testing. diff --git a/breeze b/breeze index 13685d257cc5b..d01baefe431c4 100755 --- a/breeze +++ b/breeze @@ -32,7 +32,7 @@ function manual_instructions() { echo " python -m pip install pipx" echo " pipx ensurepath" echo " pipx install -e '${MY_DIR}/dev/breeze/'" - echo " breeze setup-autocomplete --force" + echo " breeze setup autocomplete --force" echo echo " After that, both pipx and breeze should be available on your path" echo diff --git a/dev/MANUALLY_BUILDING_IMAGES.md b/dev/MANUALLY_BUILDING_IMAGES.md index 99cf589d691d4..2a07f8b0c084e 100644 --- a/dev/MANUALLY_BUILDING_IMAGES.md +++ b/dev/MANUALLY_BUILDING_IMAGES.md @@ -83,13 +83,13 @@ docker buildx ls Preparing regular images: ```shell script -breeze release-prod-images --airflow-version "${VERSION}" +breeze release-management release-prod-images --airflow-version "${VERSION}" ``` Preparing slim images: ```shell script -breeze release-prod-images --airflow-version "${VERSION}" --slim-images +breeze release-management release-prod-images --airflow-version "${VERSION}" --slim-images ``` This will wipe Breeze cache and docker-context-files in order to make sure the build is "clean". It diff --git a/dev/PROVIDER_PACKAGE_DETAILS.md b/dev/PROVIDER_PACKAGE_DETAILS.md index 34abe8f86e346..75b39e667e2f5 100644 --- a/dev/PROVIDER_PACKAGE_DETAILS.md +++ b/dev/PROVIDER_PACKAGE_DETAILS.md @@ -63,13 +63,13 @@ Details to be hashed out in [the related issue](https://github.com/apache/airflo You can verify if all providers are properly named and importable. ```bash -breeze verify-provider-packages +breeze release-management verify-provider-packages ``` You can also run the verification with an earlier airflow version to check for compatibility. ```bash -breeze verify-provider-packages --use-airflow-version 2.1.0 +breeze release-management verify-provider-packages --use-airflow-version 2.1.0 ``` @@ -78,7 +78,7 @@ breeze verify-provider-packages --use-airflow-version 2.1.0 When you want to prepare release notes for a package, you need to run: ```bash -breeze prepare-provider-documentation ... +breeze release-management prepare-provider-documentation ... ``` The version for each package is going to be updated separately for each package when we agree to the @@ -96,7 +96,7 @@ release date without providing the date (to update the existing release notes) ```bash -breeze prepare-provider-documentation google +breeze release-management prepare-provider-documentation google ``` @@ -135,7 +135,7 @@ the folders (for example Apache Hive's PACKAGE_ID is `apache.hive` ). You can se providers by running: ```bash -breeze prepare-provider-packages --help +breeze release-management prepare-provider-packages --help ``` The examples below show how you can build selected packages, but you can also build all packages by @@ -144,25 +144,25 @@ omitting the package ids altogether. * To build the release candidate packages for PyPI upload run the following command: ```bash -breeze prepare-provider-packages --package-format both --version-suffix-for-pypi=rc1 [PACKAGE_ID] ... +breeze release-management prepare-provider-packages --package-format both --version-suffix-for-pypi=rc1 [PACKAGE_ID] ... ``` for example: ```bash -breeze prepare-provider-packages --package-format both --version-suffix-for-pypi=rc1 http ... +breeze release-management prepare-provider-packages --package-format both --version-suffix-for-pypi=rc1 http ... ``` * To build the final release packages run the following command: ```bash -breeze prepare-provider-packages --package-format both [PACKAGE_ID] ... +breeze release-management prepare-provider-packages --package-format both [PACKAGE_ID] ... ``` for example: ```bash -breeze prepare-provider-packages --package-format both http ... +breeze release-management prepare-provider-packages --package-format both http ... ``` * For each package, this creates a wheel package and source distribution package in your `dist` folder with diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 674c75e955d02..98aedbcfb396b 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -269,7 +269,7 @@ The Release Candidate artifacts we vote upon should be the exact ones we vote ag - Make sure you have the latest CI image ```shell script - breeze build-image --python 3.7 + breeze ci-image build --python 3.7 ``` - Tarball the repo @@ -286,7 +286,7 @@ The Release Candidate artifacts we vote upon should be the exact ones we vote ag - Generate SHA512/ASC (If you have not generated a key yet, generate it by following instructions on http://www.apache.org/dev/openpgp.html#key-gen-generate-key) ```shell script - breeze prepare-airflow-package --package-format both + breeze release-management prepare-airflow-package --package-format both pushd dist ${AIRFLOW_REPO_ROOT}/dev/sign.sh * popd @@ -479,7 +479,7 @@ To do this we need to - Build the package: ```shell script - breeze prepare-airflow-package --version-suffix-for-pypi "${VERSION_SUFFIX}" --package-format both + breeze release-management prepare-airflow-package --version-suffix-for-pypi "${VERSION_SUFFIX}" --package-format both ``` - Verify the artifacts that would be uploaded: @@ -1013,10 +1013,10 @@ the older branches, you should set the "skip" field to true. for PYTHON in 3.7 3.8 3.9 3.10 do docker pull apache/airflow:${VERSION}-python${PYTHON} - breeze verify-prod-image --image-name apache/airflow:${VERSION}-python${PYTHON} + breeze prod-image verify --image-name apache/airflow:${VERSION}-python${PYTHON} done docker pull apache/airflow:${VERSION} -breeze verify-prod-image --image-name apache/airflow:${VERSION} +breeze prod-image verify --image-name apache/airflow:${VERSION} ``` diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md index 872e347349d7c..ab1da037c5a2b 100644 --- a/dev/README_RELEASE_PROVIDER_PACKAGES.md +++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md @@ -126,7 +126,7 @@ Details about maintaining the SEMVER version are going to be discussed and imple [the related issue](https://github.com/apache/airflow/issues/11425) ```shell script -breeze prepare-provider-documentation [packages] +breeze release-management prepare-provider-documentation [packages] ``` This command will not only prepare documentation but will also help the release manager to review @@ -148,7 +148,7 @@ When you want to regenerate the changes before the release and make sure all cha are updated, run it in non-interactive mode: ```shell script -breeze prepare-provider-documentation --answer yes [packages] +breeze release-management prepare-provider-documentation --answer yes [packages] ``` ## Build provider packages for SVN apache upload @@ -173,13 +173,13 @@ rm -rf ${AIRFLOW_REPO_ROOT}/dist/* * Release candidate packages: ```shell script -breeze prepare-provider-packages --package-format both +breeze release-management prepare-provider-packages --package-format both ``` if you only build few packages, run: ```shell script -breeze prepare-provider-packages --package-format both PACKAGE PACKAGE .... +breeze release-management prepare-provider-packages --package-format both PACKAGE PACKAGE .... ``` * Sign all your packages @@ -238,13 +238,13 @@ this will clean up dist folder before generating the packages, so you will only ```shell script rm -rf ${AIRFLOW_REPO_ROOT}/dist/* -breeze prepare-provider-packages --version-suffix-for-pypi rc1 --package-format both +breeze release-management prepare-provider-packages --version-suffix-for-pypi rc1 --package-format both ``` if you only build few packages, run: ```shell script -breeze prepare-provider-packages --version-suffix-for-pypi rc1 --package-format both PACKAGE PACKAGE .... +breeze release-management prepare-provider-packages --version-suffix-for-pypi rc1 --package-format both PACKAGE PACKAGE .... ``` * Verify the artifacts that would be uploaded: diff --git a/dev/REFRESHING_CI_CACHE.md b/dev/REFRESHING_CI_CACHE.md index 5fd458bf34a43..c6596aa8641ed 100644 --- a/dev/REFRESHING_CI_CACHE.md +++ b/dev/REFRESHING_CI_CACHE.md @@ -51,10 +51,10 @@ manual refresh might be needed. # Manually generating constraint files ```bash -breeze build-image --run-in-parallel --upgrade-to-newer-dependencies --answer yes -breeze generate-constraints --airflow-constraints-mode constraints --run-in-parallel --answer yes -breeze generate-constraints --airflow-constraints-mode constraints-source-providers --run-in-parallel --answer yes -breeze generate-constraints --airflow-constraints-mode constraints-no-providers --run-in-parallel --answer yes +breeze ci-image build --run-in-parallel --upgrade-to-newer-dependencies --answer yes +breeze release-management generate-constraints --airflow-constraints-mode constraints --run-in-parallel --answer yes +breeze release-management generate-constraints --airflow-constraints-mode constraints-source-providers --run-in-parallel --answer yes +breeze release-management generate-constraints --airflow-constraints-mode constraints-no-providers --run-in-parallel --answer yes AIRFLOW_SOURCES=$(pwd) ``` diff --git a/dev/TRACKING_BACKTRACKING_ISSUES.md b/dev/TRACKING_BACKTRACKING_ISSUES.md index 27afc390ab7a0..2a959175d3aa4 100644 --- a/dev/TRACKING_BACKTRACKING_ISSUES.md +++ b/dev/TRACKING_BACKTRACKING_ISSUES.md @@ -130,7 +130,7 @@ the version that was correctly installed before and is stored in the current con The process of tracking down which package is the "root cause" looks as follows: 1. Checkout the latest main of Airflow -2. Build the latest image (with constraints): `breeze build-image --python 3.7` +2. Build the latest image (with constraints): `breeze ci-image build --python 3.7` 3. Enter breeze `breeze` 4. Attempt to run the `pip install` command that was printed in the "Candidates ..." step 5. The command should succeed (the candidates are pinned to the "working" version) @@ -192,15 +192,15 @@ You need to install the breeze: * `pipx install -e ./dev/breeze` if you use pipx install. -Then you can run ``breeze find-newer-dependencies`` with optional flags. +Then you can run ``breeze ci find-newer-dependencies`` with optional flags. For example if you know that the build was likely broken on a given date and time (in your timezone) and you want to check python 3.8 (because this is the only failing build) you can run: ``` -breeze find-newer-dependencies --updated-on-or-after '2022-02-22 10:30:00' --timezone 'CET' --python 3.8 +breeze ci find-newer-dependencies --updated-on-or-after '2022-02-22 10:30:00' --timezone 'CET' --python 3.8 ``` The full list of options for `find-newer-dependencies` can be seen here -![breeze find-newer-dependencies](../images/breeze/output-find-newer-dependencies.svg) +![breeze ci find-newer-dependencies](../images/breeze/output-find-newer-dependencies.svg) diff --git a/dev/breeze/doc/adr/0011-unified-communication-with-the-users.md b/dev/breeze/doc/adr/0011-unified-communication-with-the-users.md index 89f967e350cd8..6de55bac57f9f 100644 --- a/dev/breeze/doc/adr/0011-unified-communication-with-the-users.md +++ b/dev/breeze/doc/adr/0011-unified-communication-with-the-users.md @@ -67,7 +67,7 @@ We are using rich and colours to communicate the type of messages: * instructions should be printed without style and default rich rendering should be used By default, we map those styles to those colors, but we can change configuration of Breeze to -be colour-blind-friendly by disabling colours in communication via `breeze config --no-colour`. +be colour-blind-friendly by disabling colours in communication via `breeze setup config --no-colour`. When signalling warning or error The communication should not be confusing and should not contain "noise". It should contain simple - usually one-liner - diff --git a/dev/breeze/doc/adr/0012-asking-user-for-confirmation.md b/dev/breeze/doc/adr/0012-asking-user-for-confirmation.md index 82dd2df35bd84..1e83095b61522 100644 --- a/dev/breeze/doc/adr/0012-asking-user-for-confirmation.md +++ b/dev/breeze/doc/adr/0012-asking-user-for-confirmation.md @@ -73,7 +73,7 @@ confirmation is necessary. Default action in this case is "quit" so if the user the operation should quit without making any change. 2) If there is a long-running (but not disruptive) operation that user directly requested (for example -building and image by `build-image` command), there should be no question asked unless there is some +building and image by `ci-image build` command), there should be no question asked unless there is some condition that could change the length of the action (for example when we realise that the user have not rebased to the latest main and rebuilding the image might take far longer than initially anticipated by the user). In case of such unexpected condition the default answer should be "quit" as well without timeout. diff --git a/dev/breeze/src/airflow_breeze/breeze.py b/dev/breeze/src/airflow_breeze/breeze.py index 608619ae8e487..c58567044f394 100755 --- a/dev/breeze/src/airflow_breeze/breeze.py +++ b/dev/breeze/src/airflow_breeze/breeze.py @@ -25,5 +25,20 @@ find_airflow_sources_root_to_operate_on() create_directories_and_files() +from airflow_breeze.commands import developer_commands # noqa +from airflow_breeze.commands.ci_commands import ci_group # noqa +from airflow_breeze.commands.ci_image_commands import ci_image # noqa +from airflow_breeze.commands.production_image_commands import prod_image # noqa +from airflow_breeze.commands.release_management_commands import release_management # noqa +from airflow_breeze.commands.setup_commands import setup # noqa +from airflow_breeze.commands.testing_commands import testing # noqa + +main.add_command(testing) +main.add_command(ci_group) +main.add_command(ci_image) +main.add_command(prod_image) +main.add_command(setup) +main.add_command(release_management) + if __name__ == '__main__': main() diff --git a/dev/breeze/src/airflow_breeze/commands/__init__.py b/dev/breeze/src/airflow_breeze/commands/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/commands/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/dev/breeze/src/airflow_breeze/commands/ci_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_commands.py index cc7089ce5d046..0baf1d8f991e6 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_commands.py @@ -24,7 +24,6 @@ import click -from airflow_breeze.commands.main_command import main from airflow_breeze.global_constants import ( DEFAULT_PYTHON_MAJOR_MINOR_VERSION, MOUNT_ALL, @@ -32,6 +31,7 @@ github_events, ) from airflow_breeze.params.shell_params import ShellParams +from airflow_breeze.utils.click_utils import BreezeGroup from airflow_breeze.utils.common_options import ( option_airflow_constraints_reference, option_answer, @@ -57,53 +57,13 @@ from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT from airflow_breeze.utils.run_utils import run_command -CI_COMMANDS = { - "name": "CI commands", - "commands": [ - "fix-ownership", - "free-space", - "resource-check", - "selective-check", - "find-newer-dependencies", - ], -} -CI_PARAMETERS = { - "breeze fix-ownership": [ - { - "name": "Fix ownership flags", - "options": [ - "--use-sudo", - ], - } - ], - "breeze selective-check": [ - { - "name": "Selective check flags", - "options": [ - "--commit-ref", - "--pr-labels", - "--default-branch", - "--github-event-name", - ], - } - ], - "breeze find-newer-dependencies": [ - { - "name": "Find newer dependencies flags", - "options": [ - "--python", - "--timezone", - "--constraints-branch", - "--updated-on-or-after", - "--max-age", - ], - } - ], -} +@click.group(cls=BreezeGroup, name='ci', help='Tools that CI workflows use to cleanup/manage CI environment') +def ci_group(): + pass -@main.command(name="free-space", help="Free space for jobs run in CI.") +@ci_group.command(name="free-space", help="Free space for jobs run in CI.") @option_verbose @option_dry_run @option_answer @@ -119,7 +79,7 @@ def free_space(verbose: bool, dry_run: bool, answer: str): run_command(["docker", "logout", "ghcr.io"], verbose=verbose, dry_run=dry_run, check=False) -@main.command(name="resource-check", help="Check if available docker resources are enough.") +@ci_group.command(name="resource-check", help="Check if available docker resources are enough.") @option_verbose @option_dry_run def resource_check(verbose: bool, dry_run: bool): @@ -168,7 +128,7 @@ def fix_ownership_without_docker(dry_run: bool, verbose: bool): fix_ownership_for_path(directory_to_fix, dry_run=dry_run, verbose=verbose) -@main.command(name="fix-ownership", help="Fix ownership of source files to be same as host user.") +@ci_group.command(name="fix-ownership", help="Fix ownership of source files to be same as host user.") @click.option( '--use-sudo', is_flag=True, @@ -236,7 +196,9 @@ def get_changed_files(commit_ref: Optional[str], dry_run: bool, verbose: bool) - return changed_files -@main.command(name="selective-check", help="Checks what kind of tests should be run for an incoming commit.") +@ci_group.command( + name="selective-check", help="Checks what kind of tests should be run for an incoming commit." +) @click.option( '--commit-ref', help="Commit-ish reference to the commit that should be checked", @@ -299,7 +261,7 @@ def selective_check( print(str(sc)) -@main.command(name="find-newer-dependencies", help="Finds which dependencies are being upgraded.") +@ci_group.command(name="find-newer-dependencies", help="Finds which dependencies are being upgraded.") @option_timezone @option_airflow_constraints_reference @option_python diff --git a/dev/breeze/src/airflow_breeze/commands/ci_commands_config.py b/dev/breeze/src/airflow_breeze/commands/ci_commands_config.py new file mode 100644 index 0000000000000..08d805ed4786d --- /dev/null +++ b/dev/breeze/src/airflow_breeze/commands/ci_commands_config.py @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from typing import Dict, List, Union + +CI_COMMANDS: Dict[str, Union[str, List[str]]] = { + "name": "CI commands", + "commands": [ + "fix-ownership", + "free-space", + "resource-check", + "selective-check", + "find-newer-dependencies", + ], +} +CI_PARAMETERS: Dict[str, List[Dict[str, Union[str, List[str]]]]] = { + "breeze ci fix-ownership": [ + { + "name": "Fix ownership flags", + "options": [ + "--use-sudo", + ], + } + ], + "breeze ci selective-check": [ + { + "name": "Selective check flags", + "options": [ + "--commit-ref", + "--pr-labels", + "--default-branch", + "--default-constraints-branch", + "--github-event-name", + ], + } + ], + "breeze ci find-newer-dependencies": [ + { + "name": "Find newer dependencies flags", + "options": [ + "--python", + "--timezone", + "--airflow-constraints-reference", + "--updated-on-or-after", + "--max-age", + ], + } + ], +} diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index 397fbc4839f88..d77d26c6a2893 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -22,9 +22,9 @@ import click -from airflow_breeze.commands.main_command import main from airflow_breeze.params.build_ci_params import BuildCiParams from airflow_breeze.params.shell_params import ShellParams +from airflow_breeze.utils.click_utils import BreezeGroup from airflow_breeze.utils.common_options import ( option_additional_dev_apt_command, option_additional_dev_apt_deps, @@ -57,8 +57,8 @@ option_parallelism, option_platform_multiple, option_prepare_buildx_cache, - option_pull_image, - option_push_image, + option_pull, + option_push, option_python, option_python_image, option_python_versions, @@ -68,7 +68,7 @@ option_tag_as_latest, option_upgrade_to_newer_dependencies, option_verbose, - option_verify_image, + option_verify, option_wait_for_image, ) from airflow_breeze.utils.confirm import STANDARD_TIMEOUT, Answer, user_confirm @@ -97,104 +97,12 @@ run_command, ) -CI_IMAGE_TOOLS_COMMANDS = { - "name": "CI Image tools", - "commands": [ - "build-image", - "pull-image", - "verify-image", - ], -} -CI_IMAGE_TOOLS_PARAMETERS = { - "breeze build-image": [ - { - "name": "Basic usage", - "options": [ - "--python", - "--upgrade-to-newer-dependencies", - "--debian-version", - "--image-tag", - "--tag-as-latest", - "--docker-cache", - "--force-build", - ], - }, - { - "name": "Building images in parallel", - "options": [ - "--run-in-parallel", - "--parallelism", - "--python-versions", - ], - }, - { - "name": "Advanced options (for power users)", - "options": [ - "--install-providers-from-sources", - "--airflow-constraints-mode", - "--airflow-constraints-reference", - "--python-image", - "--additional-python-deps", - "--runtime-apt-deps", - "--runtime-apt-command", - "--additional-extras", - "--additional-runtime-apt-deps", - "--additional-runtime-apt-env", - "--additional-runtime-apt-command", - "--additional-dev-apt-deps", - "--additional-dev-apt-env", - "--additional-dev-apt-command", - "--dev-apt-deps", - "--dev-apt-command", - ], - }, - { - "name": "Preparing cache and push (for maintainers and CI)", - "options": [ - "--github-token", - "--github-username", - "--platform", - "--login-to-github-registry", - "--push-image", - "--empty-image", - "--prepare-buildx-cache", - ], - }, - ], - "breeze pull-image": [ - { - "name": "Pull image flags", - "options": [ - "--image-tag", - "--python", - "--github-token", - "--verify-image", - "--wait-for-image", - "--tag-as-latest", - ], - }, - { - "name": "Parallel running", - "options": [ - "--run-in-parallel", - "--parallelism", - "--python-versions", - ], - }, - ], - "breeze verify-image": [ - { - "name": "Verify image flags", - "options": [ - "--image-name", - "--python", - "--image-tag", - "--pull-image", - ], - } - ], -} +@click.group( + cls=BreezeGroup, name='ci-image', help="Tools that developers can use to manually manage CI images" +) +def ci_image(): + pass def check_if_image_building_is_needed(ci_image_params: BuildCiParams, dry_run: bool, verbose: bool) -> bool: @@ -202,7 +110,7 @@ def check_if_image_building_is_needed(ci_image_params: BuildCiParams, dry_run: b if not ci_image_params.force_build and not ci_image_params.upgrade_to_newer_dependencies: if not should_we_run_the_build(build_ci_params=ci_image_params): return False - if ci_image_params.prepare_buildx_cache or ci_image_params.push_image: + if ci_image_params.prepare_buildx_cache or ci_image_params.push: login_to_github_docker_registry(image_params=ci_image_params, dry_run=dry_run, verbose=verbose) return True @@ -235,7 +143,7 @@ def start_building(params: BuildCiParams, dry_run: bool, verbose: bool): make_sure_builder_configured(parallel=True, params=params, dry_run=dry_run, verbose=verbose) -@main.command(name='build-image') +@ci_image.command(name='build') @option_github_repository @option_verbose @option_dry_run @@ -252,7 +160,7 @@ def start_building(params: BuildCiParams, dry_run: bool, verbose: bool): @option_docker_cache @option_image_tag_for_building @option_prepare_buildx_cache -@option_push_image +@option_push @option_empty_image @option_install_providers_from_sources @option_additional_extras @@ -275,7 +183,7 @@ def start_building(params: BuildCiParams, dry_run: bool, verbose: bool): @option_airflow_constraints_reference_build @option_tag_as_latest @option_additional_pip_install_flags -def build_image( +def build( verbose: bool, dry_run: bool, run_in_parallel: bool, @@ -320,7 +228,7 @@ def run_build(ci_image_params: BuildCiParams) -> None: run_build(ci_image_params=params) -@main.command(name='pull-image') +@ci_image.command(name='pull') @option_verbose @option_dry_run @option_python @@ -329,12 +237,12 @@ def run_build(ci_image_params: BuildCiParams) -> None: @option_parallelism @option_python_versions @option_github_token -@option_verify_image +@option_verify @option_wait_for_image @option_image_tag_for_pulling @option_tag_as_latest @click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED) -def pull_ci_image( +def pull( verbose: bool, dry_run: bool, python: str, @@ -346,7 +254,7 @@ def pull_ci_image( image_tag: str, wait_for_image: bool, tag_as_latest: bool, - verify_image: bool, + verify: bool, extra_pytest_args: Tuple, ): """Pull and optionally verify CI images - possibly in parallel for all Python versions.""" @@ -354,7 +262,7 @@ def pull_ci_image( get_console().print("[red]You cannot pull latest images because they are not published any more!\n") get_console().print( "[yellow]You need to specify commit tag to pull and image. If you wish to get" - " the latest image, you need to run `breeze build-image` command\n" + " the latest image, you need to run `breeze ci-image build` command\n" ) sys.exit(1) perform_environment_checks(verbose=verbose) @@ -375,7 +283,7 @@ def pull_ci_image( image_params_list=ci_image_params_list, python_version_list=python_version_list, verbose=verbose, - verify_image=verify_image, + verify=verify, wait_for_image=wait_for_image, tag_as_latest=tag_as_latest, extra_pytest_args=extra_pytest_args if extra_pytest_args is not None else (), @@ -396,8 +304,8 @@ def pull_ci_image( sys.exit(return_code) -@main.command( - name='verify-image', +@ci_image.command( + name='verify', context_settings=dict( ignore_unknown_options=True, allow_extra_args=True, @@ -409,16 +317,16 @@ def pull_ci_image( @option_github_repository @option_image_tag_for_verifying @option_image_name -@option_pull_image +@option_pull @click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED) -def verify_ci_image( +def verify( verbose: bool, dry_run: bool, python: str, github_repository: str, image_name: str, image_tag: Optional[str], - pull_image: bool, + pull: bool, extra_pytest_args: Tuple, ): """Verify CI image.""" @@ -426,7 +334,7 @@ def verify_ci_image( if image_name is None: build_params = BuildCiParams(python=python, image_tag=image_tag, github_repository=github_repository) image_name = build_params.airflow_image_name_with_tag - if pull_image: + if pull: command_to_run = ["docker", "pull", image_name] run_command(command_to_run, verbose=verbose, dry_run=dry_run, check=True) get_console().print(f"[info]Verifying CI image: {image_name}[/]") @@ -526,14 +434,14 @@ def run_build_ci_image( """ if ( ci_image_params.is_multi_platform() - and not ci_image_params.push_image + and not ci_image_params.push and not ci_image_params.prepare_buildx_cache ): get_console().print( - "\n[red]You cannot use multi-platform build without using --push-image flag or " + "\n[red]You cannot use multi-platform build without using --push flag or " "preparing buildx cache![/]\n" ) - return 1, "Error: building multi-platform image without --push-image." + return 1, "Error: building multi-platform image without --push." if verbose or dry_run: get_console().print( f"\n[info]Building CI image of airflow from {AIRFLOW_SOURCES_ROOT} " diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py new file mode 100644 index 0000000000000..4e95067484453 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from typing import Dict, List, Union + +CI_IMAGE_TOOLS_COMMANDS: Dict[str, Union[str, List[str]]] = { + "name": "CI Image tools", + "commands": [ + "build", + "pull", + "verify", + ], +} +CI_IMAGE_TOOLS_PARAMETERS: Dict[str, List[Dict[str, Union[str, List[str]]]]] = { + "breeze ci-image build": [ + { + "name": "Basic usage", + "options": [ + "--python", + "--upgrade-to-newer-dependencies", + "--debian-version", + "--image-tag", + "--tag-as-latest", + "--docker-cache", + "--force-build", + ], + }, + { + "name": "Building images in parallel", + "options": [ + "--run-in-parallel", + "--parallelism", + "--python-versions", + ], + }, + { + "name": "Advanced options (for power users)", + "options": [ + "--builder", + "--install-providers-from-sources", + "--airflow-constraints-mode", + "--airflow-constraints-reference", + "--python-image", + "--additional-python-deps", + "--runtime-apt-deps", + "--runtime-apt-command", + "--additional-extras", + "--additional-pip-install-flags", + "--additional-runtime-apt-deps", + "--additional-runtime-apt-env", + "--additional-runtime-apt-command", + "--additional-dev-apt-deps", + "--additional-dev-apt-env", + "--additional-dev-apt-command", + "--dev-apt-deps", + "--dev-apt-command", + ], + }, + { + "name": "Preparing cache and push (for maintainers and CI)", + "options": [ + "--github-token", + "--github-username", + "--platform", + "--login-to-github-registry", + "--push", + "--empty-image", + "--prepare-buildx-cache", + ], + }, + ], + "breeze ci-image pull": [ + { + "name": "Pull image flags", + "options": [ + "--image-tag", + "--python", + "--github-token", + "--verify", + "--wait-for-image", + "--tag-as-latest", + ], + }, + { + "name": "Parallel running", + "options": [ + "--run-in-parallel", + "--parallelism", + "--python-versions", + ], + }, + ], + "breeze ci-image verify": [ + { + "name": "Verify image flags", + "options": [ + "--image-name", + "--python", + "--image-tag", + "--pull", + ], + } + ], +} diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index 0910ecad3a5d6..9e4fe4bc38826 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -20,7 +20,7 @@ import sys from typing import Iterable, Optional, Tuple -import rich_click as click +import click from airflow_breeze.commands.ci_image_commands import rebuild_or_pull_ci_image_if_needed from airflow_breeze.commands.main_command import main @@ -79,154 +79,6 @@ ) from airflow_breeze.utils.visuals import ASCIIART, ASCIIART_STYLE, CHEATSHEET, CHEATSHEET_STYLE -DEVELOPER_COMMANDS = { - "name": "Developer tools", - "commands": [ - "shell", - "start-airflow", - "compile-www-assets", - "exec", - "stop", - "build-docs", - "static-checks", - ], -} - -DEVELOPER_PARAMETERS = { - "breeze": [ - { - "name": "Basic flags for the default (shell) command", - "options": [ - "--python", - "--backend", - "--postgres-version", - "--mysql-version", - "--mssql-version", - "--integration", - "--forward-credentials", - "--db-reset", - ], - }, - { - "name": "Advanced flags for the default (shell) command", - "options": [ - "--use-airflow-version", - "--constraints-reference", - "--airflow-extras", - "--use-packages-from-dist", - "--package-format", - "--force-build", - "--image-tag", - "--mount-sources", - "--debian-version", - ], - }, - ], - "breeze shell": [ - { - "name": "Basic flags", - "options": [ - "--python", - "--backend", - "--postgres-version", - "--mysql-version", - "--mssql-version", - "--integration", - "--forward-credentials", - "--db-reset", - ], - }, - { - "name": "Advanced flag for running", - "options": [ - "--use-airflow-version", - "--constraints-reference", - "--airflow-extras", - "--use-packages-from-dist", - "--package-format", - "--force-build", - "--image-tag", - "--mount-sources", - "--debian-version", - ], - }, - ], - "breeze compile-www-assets": [ - { - "name": "Compile www assets flag", - "options": [ - "--dev", - ], - } - ], - "breeze start-airflow": [ - { - "name": "Basic flags", - "options": [ - "--python", - "--load-example-dags", - "--load-default-connections", - "--backend", - "--postgres-version", - "--mysql-version", - "--mssql-version", - "--integration", - "--forward-credentials", - "--db-reset", - ], - }, - { - "name": "Advanced flag for running", - "options": [ - "--use-airflow-version", - "--constraints-reference", - "--airflow-extras", - "--use-packages-from-dist", - "--package-format", - "--force-build", - "--image-tag", - "--mount-sources", - ], - }, - ], - "breeze exec": [ - {"name": "Drops in the interactive shell of active airflow container"}, - ], - "breeze stop": [ - { - "name": "Stop flags", - "options": [ - "--preserve-volumes", - ], - }, - ], - "breeze build-docs": [ - { - "name": "Doc flags", - "options": [ - "--docs-only", - "--spellcheck-only", - "--clean-build", - "--for-production", - "--package-filter", - ], - }, - ], - "breeze static-checks": [ - { - "name": "Pre-commit flags", - "options": [ - "--type", - "--file", - "--all-files", - "--show-diff-on-failure", - "--last-commit", - ], - }, - ], -} - - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # Make sure that whatever you add here as an option is also # Added in the "main" command in breeze.py. The min command above @@ -284,7 +136,7 @@ def shell( platform: Optional[str], extra_args: Tuple, ): - """Enter breeze.py environment. this is the default command use when no other is selected.""" + """Enter breeze environment. this is the default command use when no other is selected.""" if verbose or dry_run: get_console().print("\n[success]Welcome to breeze.py[/]\n") get_console().print(f"\n[success]Root of Airflow Sources = {AIRFLOW_SOURCES_ROOT}[/]\n") @@ -366,7 +218,7 @@ def start_airflow( platform: Optional[str], extra_args: Tuple, ): - """Enter breeze.py environment and starts all Airflow components in the tmux session.""" + """Enter breeze environment and starts all Airflow components in the tmux session.""" if use_airflow_version is None: run_compile_www_assets(dev=False, verbose=verbose, dry_run=dry_run) enter_shell( @@ -599,7 +451,7 @@ def stop(verbose: bool, dry_run: bool, preserve_volumes: bool): run_command(command_to_execute, verbose=verbose, dry_run=dry_run, env=env_variables) -@main.command(name='exec', help='Joins the interactive shell of running airflow container') +@main.command(name='exec', help='Joins the interactive shell of running airflow container.') @option_verbose @option_dry_run @click.argument('exec_args', nargs=-1, type=click.UNPROCESSED) diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py new file mode 100644 index 0000000000000..5893a92d6eb51 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py @@ -0,0 +1,161 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from typing import Dict, List, Union + +DEVELOPER_COMMANDS: Dict[str, Union[str, List[str]]] = { + "name": "Basic developer commands", + "commands": [ + "start-airflow", + "static-checks", + "build-docs", + "stop", + "shell", + "exec", + "compile-www-assets", + "cleanup", + ], +} +DEVELOPER_PARAMETERS: Dict[str, List[Dict[str, Union[str, List[str]]]]] = { + "breeze": [ + { + "name": "Basic flags", + "options": [ + "--python", + "--backend", + "--postgres-version", + "--mysql-version", + "--mssql-version", + "--integration", + "--forward-credentials", + "--db-reset", + ], + }, + ], + "breeze shell": [ + { + "name": "Basic flags", + "options": [ + "--python", + "--backend", + "--postgres-version", + "--mysql-version", + "--mssql-version", + "--integration", + "--forward-credentials", + "--db-reset", + ], + }, + { + "name": "Advanced flag for running", + "options": [ + "--use-airflow-version", + "--airflow-constraints-reference", + "--platform", + "--airflow-extras", + "--use-packages-from-dist", + "--package-format", + "--force-build", + "--image-tag", + "--mount-sources", + "--debian-version", + ], + }, + ], + "breeze compile-www-assets": [ + { + "name": "Compile www assets flag", + "options": [ + "--dev", + ], + } + ], + "breeze start-airflow": [ + { + "name": "Basic flags", + "options": [ + "--python", + "--load-example-dags", + "--load-default-connections", + "--backend", + "--platform", + "--postgres-version", + "--mysql-version", + "--mssql-version", + "--integration", + "--forward-credentials", + "--db-reset", + ], + }, + { + "name": "Advanced flag for running", + "options": [ + "--use-airflow-version", + "--airflow-constraints-reference", + "--airflow-extras", + "--use-packages-from-dist", + "--package-format", + "--force-build", + "--image-tag", + "--mount-sources", + ], + }, + ], + "breeze exec": [ + {"name": "Drops in the interactive shell of active airflow container"}, + ], + "breeze stop": [ + { + "name": "Stop flags", + "options": [ + "--preserve-volumes", + ], + }, + ], + "breeze build-docs": [ + { + "name": "Doc flags", + "options": [ + "--docs-only", + "--spellcheck-only", + "--clean-build", + "--for-production", + "--package-filter", + ], + }, + ], + "breeze static-checks": [ + { + "name": "Pre-commit flags", + "options": [ + "--type", + "--file", + "--all-files", + "--show-diff-on-failure", + "--last-commit", + "--commit-ref", + ], + }, + ], + "breeze cleanup": [ + { + "name": "Cleanup flags", + "options": [ + "--all", + ], + }, + ], +} diff --git a/dev/breeze/src/airflow_breeze/commands/main_command.py b/dev/breeze/src/airflow_breeze/commands/main_command.py index fbf3b38067684..f4adab3c15187 100644 --- a/dev/breeze/src/airflow_breeze/commands/main_command.py +++ b/dev/breeze/src/airflow_breeze/commands/main_command.py @@ -15,52 +15,91 @@ # specific language governing permissions and limitations # under the License. import platform +import shutil import subprocess import sys +from typing import Optional +from click import Context + +from airflow_breeze.commands.ci_image_commands import ci_image +from airflow_breeze.commands.production_image_commands import prod_image +from airflow_breeze.commands.testing_commands import testing from airflow_breeze.configure_rich_click import click +from airflow_breeze.utils.click_utils import BreezeGroup from airflow_breeze.utils.common_options import ( - option_airflow_extras, option_answer, option_backend, option_db_reset, - option_debian_version, option_dry_run, - option_force_build, option_forward_credentials, option_github_repository, - option_installation_package_format, option_integration, - option_mount_sources, option_mssql_version, option_mysql_version, option_postgres_version, option_python, - option_use_airflow_version, - option_use_packages_from_dist, option_verbose, ) +from airflow_breeze.utils.confirm import Answer, user_confirm +from airflow_breeze.utils.console import get_console +from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR +from airflow_breeze.utils.run_utils import run_command -@click.group(invoke_without_command=True, context_settings={'help_option_names': ['-h', '--help']}) -@option_verbose -@option_dry_run +def print_deprecated(deprecated_command: str, command_to_use: str): + get_console().print("[yellow]" + ("#" * 80) + "\n") + get_console().print(f"[yellow]The command '{deprecated_command}' is deprecated!\n") + get_console().print(f"Use 'breeze {command_to_use}' instead\n") + get_console().print("[yellow]" + ("#" * 80) + "\n") + + +class MainGroupWithAliases(BreezeGroup): + def get_command(self, ctx: Context, cmd_name: str): + # Aliases for important commands moved to sub-commands + from airflow_breeze.commands.setup_commands import setup + + rv = click.Group.get_command(self, ctx, cmd_name) + if rv is not None: + return rv + if cmd_name == 'build-image': + print_deprecated('build-image', 'ci-image build') + return ci_image.get_command(ctx, 'build') + if cmd_name == 'build-prod-image': + print_deprecated('build-prod-image', 'prod-image build') + return prod_image.get_command(ctx, 'build') + if cmd_name == 'tests': + print_deprecated('tests', 'testing tests') + return testing.get_command(ctx, 'tests') + if cmd_name == 'config': + print_deprecated('config', 'setup config') + return setup.get_command(ctx, 'config') + if cmd_name == 'setup-autocomplete': + print_deprecated('setup-autocomplete', 'setup autocomplete') + return setup.get_command(ctx, 'autocomplete') + if cmd_name == 'version': + # version alias does not need to be deprecated. It's ok to keep it also at top level + # even if it is not displayed in help + return setup.get_command(ctx, 'version') + return None + + +@click.group( + cls=MainGroupWithAliases, + invoke_without_command=True, + context_settings={'help_option_names': ['-h', '--help']}, +) @option_python -@option_github_repository @option_backend @option_postgres_version @option_mysql_version @option_mssql_version -@option_debian_version -@option_forward_credentials -@option_force_build -@option_use_airflow_version -@option_airflow_extras -@option_use_packages_from_dist -@option_installation_package_format -@option_mount_sources @option_integration +@option_forward_credentials @option_db_reset +@option_verbose +@option_dry_run +@option_github_repository @option_answer @click.pass_context def main(ctx: click.Context, **kwargs): @@ -140,3 +179,77 @@ def check_for_rosetta_environment(): pass except PermissionError: pass + + +@main.command( + name="cleanup", + help="Cleans the cache of parameters, docker cache and optionally built CI/PROD images.", +) +@click.option( + '--all', + is_flag=True, + help='Also remove currently downloaded Breeze images.', +) +@option_verbose +@option_answer +@option_dry_run +@option_github_repository +def cleanup(verbose: bool, dry_run: bool, github_repository: str, all: bool, answer: Optional[str]): + if all: + get_console().print( + "\n[info]Removing cache of parameters, clean up docker cache " + "and remove locally downloaded images[/]" + ) + else: + get_console().print("[info]Removing cache of parameters, and cleans up docker cache[/]") + if all: + docker_images_command_to_execute = [ + 'docker', + 'images', + '--filter', + 'label=org.apache.airflow.image', + '--format', + '{{.Repository}}:{{.Tag}}', + ] + command_result = run_command( + docker_images_command_to_execute, verbose=verbose, text=True, capture_output=True + ) + images = command_result.stdout.splitlines() if command_result and command_result.stdout else [] + if images: + get_console().print("[info]Removing images:[/]") + for image in images: + get_console().print(f"[info] * {image}[/]") + get_console().print() + docker_rmi_command_to_execute = [ + 'docker', + 'rmi', + '--force', + ] + docker_rmi_command_to_execute.extend(images) + given_answer = user_confirm("Are you sure with the removal?") + if given_answer == Answer.YES: + run_command(docker_rmi_command_to_execute, verbose=verbose, dry_run=dry_run, check=False) + elif given_answer == Answer.QUIT: + sys.exit(0) + else: + get_console().print("[info]No locally downloaded images to remove[/]\n") + get_console().print("Pruning docker images") + given_answer = user_confirm("Are you sure with the removal?") + if given_answer == Answer.YES: + system_prune_command_to_execute = ['docker', 'system', 'prune'] + run_command( + system_prune_command_to_execute, + verbose=verbose, + dry_run=dry_run, + check=False, + enabled_output_group=True, + ) + elif given_answer == Answer.QUIT: + sys.exit(0) + get_console().print(f"Removing build cache dir ${BUILD_CACHE_DIR}") + given_answer = user_confirm("Are you sure with the removal?") + if given_answer == Answer.YES: + if not dry_run: + shutil.rmtree(BUILD_CACHE_DIR, ignore_errors=True) + elif given_answer == Answer.QUIT: + sys.exit(0) diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index 2b75729789a09..eb2de5c03d8b2 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -22,9 +22,9 @@ import click -from airflow_breeze.commands.main_command import main from airflow_breeze.global_constants import ALLOWED_INSTALLATION_METHODS, DEFAULT_EXTRAS from airflow_breeze.params.build_prod_params import BuildProdParams +from airflow_breeze.utils.click_utils import BreezeGroup from airflow_breeze.utils.common_options import ( option_additional_dev_apt_command, option_additional_dev_apt_deps, @@ -56,8 +56,8 @@ option_parallelism, option_platform_multiple, option_prepare_buildx_cache, - option_pull_image, - option_push_image, + option_pull, + option_push, option_python, option_python_image, option_python_versions, @@ -67,7 +67,7 @@ option_tag_as_latest, option_upgrade_to_newer_dependencies, option_verbose, - option_verify_image, + option_verify, option_wait_for_image, ) from airflow_breeze.utils.console import get_console @@ -88,118 +88,6 @@ from airflow_breeze.utils.run_tests import verify_an_image from airflow_breeze.utils.run_utils import filter_out_none, fix_group_permissions, run_command -PRODUCTION_IMAGE_TOOLS_COMMANDS = { - "name": "Production Image tools", - "commands": [ - "build-prod-image", - "pull-prod-image", - "verify-prod-image", - ], -} -PRODUCTION_IMAGE_TOOLS_PARAMETERS = { - "breeze build-prod-image": [ - { - "name": "Basic usage", - "options": [ - "--python", - "--install-airflow-version", - "--upgrade-to-newer-dependencies", - "--debian-version", - "--image-tag", - "--tag-as-latest", - "--docker-cache", - ], - }, - { - "name": "Building images in parallel", - "options": [ - "--run-in-parallel", - "--parallelism", - "--python-versions", - ], - }, - { - "name": "Options for customizing images", - "options": [ - "--install-providers-from-sources", - "--airflow-extras", - "--airflow-constraints-mode", - "--airflow-constraints-reference", - "--python-image", - "--additional-python-deps", - "--additional-extras", - "--additional-runtime-apt-deps", - "--additional-runtime-apt-env", - "--additional-runtime-apt-command", - "--additional-dev-apt-deps", - "--additional-dev-apt-env", - "--additional-dev-apt-command", - "--runtime-apt-deps", - "--runtime-apt-command", - "--dev-apt-deps", - "--dev-apt-command", - ], - }, - { - "name": "Customization options (for specific customization needs)", - "options": [ - "--install-packages-from-context", - "--cleanup-context", - "--disable-mysql-client-installation", - "--disable-mssql-client-installation", - "--disable-postgres-client-installation", - "--disable-airflow-repo-cache", - "--install-airflow-reference", - "--installation-method", - ], - }, - { - "name": "Preparing cache and push (for maintainers and CI)", - "options": [ - "--github-token", - "--github-username", - "--platform", - "--login-to-github-registry", - "--push-image", - "--empty-image", - "--prepare-buildx-cache", - ], - }, - ], - "breeze pull-prod-image": [ - { - "name": "Pull image flags", - "options": [ - "--image-tag", - "--python", - "--github-token", - "--verify-image", - "--wait-for-image", - "--tag-as-latest", - ], - }, - { - "name": "Parallel running", - "options": [ - "--run-in-parallel", - "--parallelism", - "--python-versions", - ], - }, - ], - "breeze verify-prod-image": [ - { - "name": "Verify image flags", - "options": [ - "--image-name", - "--python", - "--image-tag", - "--pull-image", - ], - } - ], -} - def start_building(parallel: bool, prod_image_params: BuildProdParams, dry_run: bool, verbose: bool): make_sure_builder_configured( @@ -208,7 +96,7 @@ def start_building(parallel: bool, prod_image_params: BuildProdParams, dry_run: if prod_image_params.cleanup_context: clean_docker_context_files(verbose=verbose, dry_run=dry_run) check_docker_context_files(prod_image_params.install_packages_from_context) - if prod_image_params.prepare_buildx_cache or prod_image_params.push_image: + if prod_image_params.prepare_buildx_cache or prod_image_params.push: login_to_github_docker_registry(image_params=prod_image_params, dry_run=dry_run, verbose=verbose) @@ -240,10 +128,17 @@ def run_build_in_parallel( pool.close() +@click.group( + cls=BreezeGroup, name='prod-image', help="Tools that developers can use to manually manage PROD images" +) +def prod_image(): + pass + + @option_verbose @option_dry_run @option_answer -@main.command(name='build-prod-image') +@prod_image.command(name='build') @option_python @option_run_in_parallel @option_parallelism @@ -257,7 +152,7 @@ def run_build_in_parallel( @option_docker_cache @option_image_tag_for_building @option_prepare_buildx_cache -@option_push_image +@option_push @option_empty_image @option_airflow_constraints_mode_prod @click.option( @@ -314,7 +209,7 @@ def run_build_in_parallel( @option_runtime_apt_deps @option_tag_as_latest @option_additional_pip_install_flags -def build_prod_image( +def build( verbose: bool, dry_run: bool, run_in_parallel: bool, @@ -361,7 +256,7 @@ def run_build(prod_image_params: BuildProdParams) -> None: run_build(prod_image_params=params) -@main.command(name='pull-prod-image') +@prod_image.command(name='pull') @option_verbose @option_dry_run @option_python @@ -373,7 +268,7 @@ def run_build(prod_image_params: BuildProdParams) -> None: @option_image_tag_for_pulling @option_wait_for_image @option_tag_as_latest -@option_verify_image +@option_verify @click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED) def pull_prod_image( verbose: bool, @@ -387,7 +282,7 @@ def pull_prod_image( image_tag: str, wait_for_image: bool, tag_as_latest: bool, - verify_image: bool, + verify: bool, extra_pytest_args: Tuple, ): """Pull and optionally verify Production images - possibly in parallel for all Python versions.""" @@ -395,7 +290,7 @@ def pull_prod_image( get_console().print("[red]You cannot pull latest images because they are not published any more!\n") get_console().print( "[yellow]You need to specify commit tag to pull and image. If you wish to get" - " the latest image, you need to run `breeze build-image` command\n" + " the latest image, you need to run `breeze ci-image build` command\n" ) sys.exit(1) perform_environment_checks(verbose=verbose) @@ -416,7 +311,7 @@ def pull_prod_image( image_params_list=prod_image_params_list, python_version_list=python_version_list, verbose=verbose, - verify_image=verify_image, + verify=verify, wait_for_image=wait_for_image, tag_as_latest=tag_as_latest, extra_pytest_args=extra_pytest_args if extra_pytest_args is not None else (), @@ -439,8 +334,8 @@ def pull_prod_image( sys.exit(return_code) -@main.command( - name='verify-prod-image', +@prod_image.command( + name='verify', context_settings=dict( ignore_unknown_options=True, allow_extra_args=True, @@ -452,21 +347,21 @@ def pull_prod_image( @option_github_repository @option_image_tag_for_verifying @option_image_name -@option_pull_image +@option_pull @click.option( '--slim-image', help='The image to verify is slim and non-slim tests should be skipped.', is_flag=True, ) @click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED) -def verify_prod_image( +def verify( verbose: bool, dry_run: bool, python: str, github_repository: str, image_name: str, image_tag: Optional[str], - pull_image: bool, + pull: bool, slim_image: bool, extra_pytest_args: Tuple, ): @@ -477,7 +372,7 @@ def verify_prod_image( python=python, image_tag=image_tag, github_repository=github_repository ) image_name = build_params.airflow_image_name_with_tag - if pull_image: + if pull: command_to_run = ["docker", "pull", image_name] run_command(command_to_run, verbose=verbose, dry_run=dry_run, check=True) get_console().print(f"[info]Verifying PROD image: {image_name}[/]") @@ -562,17 +457,18 @@ def run_build_production_image( :param verbose: print commands when running :param dry_run: do not execute "write" commands - just print what would happen :param prod_image_params: PROD image parameters + :param parallel: run build in parallel """ if ( prod_image_params.is_multi_platform() - and not prod_image_params.push_image + and not prod_image_params.push and not prod_image_params.prepare_buildx_cache ): get_console().print( - "\n[red]You cannot use multi-platform build without using --push-image flag" + "\n[red]You cannot use multi-platform build without using --push flag" " or preparing buildx cache![/]\n" ) - return 1, "Error: building multi-platform image without --push-image." + return 1, "Error: building multi-platform image without --push." get_console().print(f"\n[info]Building PROD Image for Python {prod_image_params.python}\n") if prod_image_params.prepare_buildx_cache: build_command_result = build_cache( diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py new file mode 100644 index 0000000000000..049086c82cc38 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from typing import Dict, List, Union + +PRODUCTION_IMAGE_TOOLS_COMMANDS: Dict[str, Union[str, List[str]]] = { + "name": "Production Image tools", + "commands": [ + "build", + "pull", + "verify", + ], +} +PRODUCTION_IMAGE_TOOLS_PARAMETERS: Dict[str, List[Dict[str, Union[str, List[str]]]]] = { + "breeze prod-image build": [ + { + "name": "Basic usage", + "options": [ + "--python", + "--install-airflow-version", + "--upgrade-to-newer-dependencies", + "--debian-version", + "--image-tag", + "--tag-as-latest", + "--docker-cache", + ], + }, + { + "name": "Building images in parallel", + "options": [ + "--run-in-parallel", + "--parallelism", + "--python-versions", + ], + }, + { + "name": "Options for customizing images", + "options": [ + "--builder", + "--install-providers-from-sources", + "--airflow-extras", + "--airflow-constraints-mode", + "--airflow-constraints-reference", + "--python-image", + "--additional-extras", + "--additional-pip-install-flags", + "--additional-python-deps", + "--additional-runtime-apt-deps", + "--additional-runtime-apt-env", + "--additional-runtime-apt-command", + "--additional-dev-apt-deps", + "--additional-dev-apt-env", + "--additional-dev-apt-command", + "--runtime-apt-deps", + "--runtime-apt-command", + "--dev-apt-deps", + "--dev-apt-command", + ], + }, + { + "name": "Customization options (for specific customization needs)", + "options": [ + "--install-packages-from-context", + "--cleanup-context", + "--disable-mysql-client-installation", + "--disable-mssql-client-installation", + "--disable-postgres-client-installation", + "--disable-airflow-repo-cache", + "--install-airflow-reference", + "--installation-method", + ], + }, + { + "name": "Preparing cache and push (for maintainers and CI)", + "options": [ + "--github-token", + "--github-username", + "--platform", + "--login-to-github-registry", + "--push", + "--empty-image", + "--prepare-buildx-cache", + ], + }, + ], + "breeze prod-image pull": [ + { + "name": "Pull image flags", + "options": [ + "--image-tag", + "--python", + "--github-token", + "--verify", + "--wait-for-image", + "--tag-as-latest", + ], + }, + { + "name": "Parallel running", + "options": [ + "--run-in-parallel", + "--parallelism", + "--python-versions", + ], + }, + ], + "breeze prod-image verify": [ + { + "name": "Verify image flags", + "options": [ + "--image-name", + "--python", + "--slim-image", + "--image-tag", + "--pull", + ], + } + ], +} diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 12aea0ca5ede9..db05b80a0b2c2 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -25,7 +25,6 @@ import click from airflow_breeze.commands.ci_image_commands import rebuild_or_pull_ci_image_if_needed -from airflow_breeze.commands.main_command import main from airflow_breeze.global_constants import ( ALLOWED_PLATFORMS, CURRENT_PYTHON_MAJOR_MINOR_VERSIONS, @@ -35,6 +34,7 @@ MULTI_PLATFORM, ) from airflow_breeze.params.shell_params import ShellParams +from airflow_breeze.utils.click_utils import BreezeGroup from airflow_breeze.utils.common_options import ( argument_packages, option_airflow_constraints_mode_ci, @@ -67,89 +67,6 @@ from airflow_breeze.utils.python_versions import get_python_version_list from airflow_breeze.utils.run_utils import RunCommandResult, run_command, run_compile_www_assets -RELEASE_MANAGEMENT_PARAMETERS = { - "breeze prepare-airflow-package": [ - {"name": "Package flags", "options": ["--package-format", "--version-suffix-for-pypi"]} - ], - "breeze verify-provider-packages": [ - { - "name": "Provider verification flags", - "options": [ - "--use-airflow-version", - "--airflow-constraints-reference", - "--airflow-extras", - "--use-packages-from-dist", - "--package-format", - "--skip-constraints", - "--debug", - ], - } - ], - "breeze prepare-provider-packages": [ - { - "name": "Package flags", - "options": [ - "--package-format", - "--version-suffix-for-pypi", - "--package-list-file", - "--debug", - ], - } - ], - "breeze prepare-provider-documentation": [ - { - "name": "Provider documentation preparation flags", - "options": [ - "--debug", - ], - } - ], - "breeze generate-constraints": [ - { - "name": "Generate constraints flags", - "options": [ - "--image-tag", - "--python", - "--airflow-constraints-mode", - "--debug", - ], - }, - { - "name": "Parallel running", - "options": [ - "--run-in-parallel", - "--parallelism", - "--python-versions", - ], - }, - ], - "breeze release-prod-images": [ - { - "name": "Release PROD IMAGE flags", - "options": [ - "--airflow-version", - "--dockerhub-repo", - "--slim-images", - "--limit-python", - "--limit-platform", - "--skip-latest", - ], - } - ], -} - -RELEASE_MANAGEMENT_COMMANDS = { - "name": "Release management", - "commands": [ - "verify-provider-packages", - "prepare-provider-documentation", - "prepare-provider-packages", - "prepare-airflow-package", - "release-prod-images", - "generate-constraints", - ], -} - option_debug_release_management = click.option( "--debug", is_flag=True, @@ -215,7 +132,16 @@ def run_with_debug( ) -@main.command( +@click.group( + cls=BreezeGroup, + name='release-management', + help="Tools that release managers can use to prepare and manage Airflow releases", +) +def release_management(): + pass + + +@release_management.command( name='prepare-airflow-package', help="Prepare sdist/whl package of Airflow.", ) @@ -257,7 +183,7 @@ def prepare_airflow_packages( sys.exit(result_command.returncode) -@main.command( +@release_management.command( name='prepare-provider-documentation', help="Prepare CHANGELOG, README and COMMITS information for providers.", ) @@ -297,7 +223,7 @@ def prepare_provider_documentation( sys.exit(result_command.returncode) -@main.command( +@release_management.command( name='prepare-provider-packages', help="Prepare sdist/whl packages of Airflow Providers.", ) @@ -392,7 +318,7 @@ def run_generate_constraints_in_parallel( pool.close() -@main.command( +@release_management.command( name='generate-constraints', help="Generates pinned constraint files with all extras from setup.py in parallel.", ) @@ -436,7 +362,7 @@ def generate_constraints( if run_in_parallel: get_console().print("\n[info]Use this command to build the images:[/]\n") get_console().print( - f" breeze build-image --run-in-parallel --python-versions '{python_versions}' " + f" breeze ci-image build --run-in-parallel --python-versions '{python_versions}' " f"--upgrade-to-newer-dependencies\n" ) else: @@ -445,7 +371,7 @@ def generate_constraints( ) get_console().print("\n[info]Use this command to build the image:[/]\n") get_console().print( - f" breeze build-image --python '{shell_params.python}' " + f" breeze ci-image build --python '{shell_params.python}' " f"--upgrade-to-newer-dependencies\n" ) sys.exit(1) @@ -489,7 +415,7 @@ def generate_constraints( sys.exit(return_code) -@main.command( +@release_management.command( name='verify-provider-packages', help="Verifies if all provider code is following expectations for providers.", ) @@ -566,7 +492,7 @@ def alias_image(image_from: str, image_to: str, dry_run: bool, verbose: bool): ) -@main.command( +@release_management.command( name="release-prod-images", help="Release production images to DockerHub (needs DockerHub permissions)." ) @click.option('--airflow-version', required=True, help="Airflow version to release (2.3.0, 2.3.0rc1 etc.)") diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py new file mode 100644 index 0000000000000..641b181eacd85 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py @@ -0,0 +1,100 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from typing import Dict, List, Union + +RELEASE_MANAGEMENT_COMMANDS: Dict[str, Union[str, List[str]]] = { + "name": "Release management", + "commands": [ + "verify-provider-packages", + "prepare-provider-documentation", + "prepare-provider-packages", + "prepare-airflow-package", + "release-prod-images", + "generate-constraints", + ], +} + +RELEASE_MANAGEMENT_PARAMETERS: Dict[str, List[Dict[str, Union[str, List[str]]]]] = { + "breeze release-management prepare-airflow-package": [ + {"name": "Package flags", "options": ["--package-format", "--version-suffix-for-pypi"]} + ], + "breeze release-management verify-provider-packages": [ + { + "name": "Provider verification flags", + "options": [ + "--use-airflow-version", + "--airflow-constraints-reference", + "--airflow-extras", + "--use-packages-from-dist", + "--package-format", + "--skip-constraints", + "--debug", + ], + } + ], + "breeze release-management prepare-provider-packages": [ + { + "name": "Package flags", + "options": [ + "--package-format", + "--version-suffix-for-pypi", + "--package-list-file", + "--debug", + ], + } + ], + "breeze release-management prepare-provider-documentation": [ + { + "name": "Provider documentation preparation flags", + "options": [ + "--debug", + ], + } + ], + "breeze release-management generate-constraints": [ + { + "name": "Generate constraints flags", + "options": [ + "--image-tag", + "--python", + "--airflow-constraints-mode", + "--debug", + ], + }, + { + "name": "Parallel running", + "options": [ + "--run-in-parallel", + "--parallelism", + "--python-versions", + ], + }, + ], + "breeze release-management release-prod-images": [ + { + "name": "Release PROD IMAGE flags", + "options": [ + "--airflow-version", + "--dockerhub-repo", + "--slim-images", + "--limit-python", + "--limit-platform", + "--skip-latest", + ], + } + ], +} diff --git a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py b/dev/breeze/src/airflow_breeze/commands/setup_commands.py similarity index 75% rename from dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py rename to dev/breeze/src/airflow_breeze/commands/setup_commands.py index 7ef0a11a2074b..f9d6d488db2e4 100644 --- a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/setup_commands.py @@ -29,11 +29,11 @@ from airflow_breeze import NAME, VERSION from airflow_breeze.commands.main_command import main from airflow_breeze.utils.cache import check_if_cache_exists, delete_cache, touch_cache_file +from airflow_breeze.utils.click_utils import BreezeGroup from airflow_breeze.utils.common_options import ( option_answer, option_backend, option_dry_run, - option_github_repository, option_mssql_version, option_mysql_version, option_postgres_version, @@ -41,11 +41,10 @@ option_verbose, ) from airflow_breeze.utils.confirm import STANDARD_TIMEOUT, Answer, user_confirm -from airflow_breeze.utils.console import get_console +from airflow_breeze.utils.console import get_console, get_stderr_console from airflow_breeze.utils.docker_command_utils import perform_environment_checks from airflow_breeze.utils.path_utils import ( AIRFLOW_SOURCES_ROOT, - BUILD_CACHE_DIR, get_installation_airflow_sources, get_installation_sources_config_metadata_hash, get_package_setup_metadata_hash, @@ -57,132 +56,10 @@ from airflow_breeze.utils.run_utils import assert_pre_commit_installed, run_command from airflow_breeze.utils.visuals import ASCIIART, ASCIIART_STYLE -CONFIGURATION_AND_MAINTENANCE_COMMANDS = { - "name": "Configuration & maintenance", - "commands": [ - "cleanup", - "self-upgrade", - "setup-autocomplete", - "config", - "regenerate-command-images", - "command-hash-export", - "version", - ], -} -CONFIGURATION_AND_MAINTENANCE_PARAMETERS = { - "breeze cleanup": [ - { - "name": "Cleanup flags", - "options": [ - "--all", - ], - }, - ], - "breeze self-upgrade": [ - { - "name": "Self-upgrade flags", - "options": [ - "--use-current-airflow-sources", - "--force", - ], - } - ], - "breeze setup-autocomplete": [ - { - "name": "Setup autocomplete flags", - "options": [ - "--force", - ], - }, - ], - "breeze config": [ - { - "name": "Config flags", - "options": [ - "--python", - "--backend", - "--cheatsheet", - "--asciiart", - "--colour", - ], - }, - ], -} - - -@main.command( - name="cleanup", - help="Cleans the cache of parameters, docker cache and optionally - currently downloaded images.", -) -@click.option( - '--all', - is_flag=True, - help='Also remove currently downloaded Breeze images.', -) -@option_verbose -@option_answer -@option_dry_run -@option_github_repository -def cleanup(verbose: bool, dry_run: bool, github_repository: str, all: bool, answer: Optional[str]): - if all: - get_console().print( - "\n[info]Removing cache of parameters, clean up docker cache " - "and remove locally downloaded images[/]" - ) - else: - get_console().print("[info]Removing cache of parameters, and cleans up docker cache[/]") - if all: - docker_images_command_to_execute = [ - 'docker', - 'images', - '--filter', - 'label=org.apache.airflow.image', - '--format', - '{{.Repository}}:{{.Tag}}', - ] - command_result = run_command( - docker_images_command_to_execute, verbose=verbose, text=True, capture_output=True - ) - images = command_result.stdout.splitlines() if command_result and command_result.stdout else [] - if images: - get_console().print("[info]Removing images:[/]") - for image in images: - get_console().print(f"[info] * {image}[/]") - get_console().print() - docker_rmi_command_to_execute = [ - 'docker', - 'rmi', - '--force', - ] - docker_rmi_command_to_execute.extend(images) - given_answer = user_confirm("Are you sure with the removal?") - if given_answer == Answer.YES: - run_command(docker_rmi_command_to_execute, verbose=verbose, dry_run=dry_run, check=False) - elif given_answer == Answer.QUIT: - sys.exit(0) - else: - get_console().print("[info]No locally downloaded images to remove[/]\n") - get_console().print("Pruning docker images") - given_answer = user_confirm("Are you sure with the removal?") - if given_answer == Answer.YES: - system_prune_command_to_execute = ['docker', 'system', 'prune'] - run_command( - system_prune_command_to_execute, - verbose=verbose, - dry_run=dry_run, - check=False, - enabled_output_group=True, - ) - elif given_answer == Answer.QUIT: - sys.exit(0) - get_console().print(f"Removing build cache dir ${BUILD_CACHE_DIR}") - given_answer = user_confirm("Are you sure with the removal?") - if given_answer == Answer.YES: - if not dry_run: - shutil.rmtree(BUILD_CACHE_DIR, ignore_errors=True) - elif given_answer == Answer.QUIT: - sys.exit(0) +@click.group(cls=BreezeGroup, name='setup', help='Tools that developers can use to configure Breeze') +def setup(): + pass @click.option( @@ -198,7 +75,7 @@ def cleanup(verbose: bool, dry_run: bool, github_repository: str, all: bool, ans help='Use current workdir Airflow sources for upgrade' + (f" rather than from {get_installation_airflow_sources()}." if not output_file_for_recording else "."), ) -@main.command( +@setup.command( name='self-upgrade', help="Self upgrade Breeze. By default it re-installs Breeze " f"from {get_installation_airflow_sources()}." @@ -230,8 +107,8 @@ def self_upgrade(force: bool, use_current_airflow_sources: bool): help='Force autocomplete setup even if already setup before (overrides the setup).', ) @option_answer -@main.command(name='setup-autocomplete') -def setup_autocomplete(verbose: bool, dry_run: bool, force: bool, answer: Optional[str]): +@setup.command(name='autocomplete') +def autocomplete(verbose: bool, dry_run: bool, force: bool, answer: Optional[str]): """ Enables autocompletion of breeze commands. """ @@ -289,7 +166,7 @@ def setup_autocomplete(verbose: bool, dry_run: bool, force: bool, answer: Option @option_verbose -@main.command() +@setup.command() def version(verbose: bool): """Print information about version of apache-airflow-breeze.""" @@ -310,7 +187,7 @@ def version(verbose: bool): ) -@main.command(name='config') +@setup.command(name='config') @option_python @option_backend @option_postgres_version @@ -391,7 +268,7 @@ def dict_hash(dictionary: Dict[str, Any]) -> str: return dhash.hexdigest() -@main.command( +@setup.command( name="command-hash-export", help="Outputs hash of all click commands to file or stdout if `-` " "is used (useful to see if images should be regenerated).", @@ -402,11 +279,16 @@ def command_hash_export(verbose: bool, output: IO): with Context(main) as ctx: the_context_dict = ctx.to_info_dict() if verbose: - get_console().print(the_context_dict) + get_stderr_console().print(the_context_dict) output.write(f"main:{dict_hash(the_context_dict['command']['params'])}\n") commands_dict = the_context_dict['command']['commands'] for command in sorted(commands_dict.keys()): - output.write(f"{command}:{dict_hash(commands_dict[command])}\n") + current_command_dict = commands_dict[command] + if 'commands' in current_command_dict: + for subcommand in sorted(current_command_dict.keys()): + output.write(f"{command}:{subcommand}:{dict_hash(current_command_dict[subcommand])}\n") + else: + output.write(f"{command}:{dict_hash(current_command_dict)}\n") def write_to_shell(command_to_execute: str, dry_run: bool, script_path: str, force_setup: bool) -> bool: @@ -476,7 +358,7 @@ def backup(script_path_file: Path): shutil.copy(str(script_path_file), str(script_path_file) + ".bak") -@main.command(name="regenerate-command-images", help="Regenerate breeze command images.") +@setup.command(name="regenerate-command-images", help="Regenerate breeze command images.") @option_verbose @option_dry_run def regenerate_command_images(verbose: bool, dry_run: bool): diff --git a/dev/breeze/src/airflow_breeze/commands/setup_commands_config.py b/dev/breeze/src/airflow_breeze/commands/setup_commands_config.py new file mode 100644 index 0000000000000..b4560a1b217ac --- /dev/null +++ b/dev/breeze/src/airflow_breeze/commands/setup_commands_config.py @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from typing import Dict, List, Union + +SETUP_COMMANDS: Dict[str, Union[str, List[str]]] = { + "name": "Setup", + "commands": [ + "autocomplete", + "self-upgrade", + "cleanup", + "config", + "regenerate-command-images", + "command-hash-export", + "version", + ], +} +SETUP_PARAMETERS: Dict[str, List[Dict[str, Union[str, List[str]]]]] = { + "breeze setup self-upgrade": [ + { + "name": "Self-upgrade flags", + "options": [ + "--use-current-airflow-sources", + "--force", + ], + } + ], + "breeze setup autocomplete": [ + { + "name": "Setup autocomplete flags", + "options": [ + "--force", + ], + }, + ], + "breeze setup config": [ + { + "name": "Config flags", + "options": [ + "--python", + "--backend", + "--postgres-version", + "--mysql-version", + "--mssql-version", + "--cheatsheet", + "--asciiart", + "--colour", + ], + }, + ], +} diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index 05aa3aa7e8fb6..1427e4551a027 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -27,11 +27,11 @@ import click -from airflow_breeze.commands.main_command import main from airflow_breeze.global_constants import ALLOWED_TEST_TYPE_CHOICES from airflow_breeze.params.build_prod_params import BuildProdParams from airflow_breeze.params.shell_params import ShellParams from airflow_breeze.utils.ci_group import ci_group +from airflow_breeze.utils.click_utils import BreezeGroup from airflow_breeze.utils.common_options import ( option_backend, option_db_reset, @@ -56,49 +56,13 @@ from airflow_breeze.utils.run_tests import run_docker_compose_tests from airflow_breeze.utils.run_utils import RunCommandResult, run_command -TESTING_COMMANDS = { - "name": "Testing", - "commands": ["docker-compose-tests", "tests"], -} -TESTING_PARAMETERS = { - "breeze docker-compose-tests": [ - { - "name": "Docker-compose tests flag", - "options": [ - "--image-name", - "--image-tag", - "--python", - ], - } - ], - "breeze tests": [ - { - "name": "Basic flag for tests command", - "options": [ - "--integration", - "--test-type", - "--db-reset", - "--backend", - "--python", - "--postgres-version", - "--mysql-version", - "--mssql-version", - ], - }, - { - "name": "Advanced flag for tests command", - "options": [ - "--limit-progress-output", - "--image-tag", - "--mount-sources", - ], - }, - ], -} +@click.group(cls=BreezeGroup, name='testing', help='Tools that developers can use to run tests') +def testing(): + pass -@main.command( +@testing.command( name='docker-compose-tests', context_settings=dict( ignore_unknown_options=True, @@ -222,7 +186,7 @@ def run_with_progress( return result -@main.command( +@testing.command( name='tests', help="Run the specified unit test targets. Multiple targets may be specified separated by spaces.", context_settings=dict( diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py new file mode 100644 index 0000000000000..c91b336f7afb0 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from typing import Dict, List, Union + +TESTING_COMMANDS: Dict[str, Union[str, List[str]]] = { + "name": "Testing", + "commands": ["tests", "docker-compose-tests"], +} +TESTING_PARAMETERS: Dict[str, List[Dict[str, Union[str, List[str]]]]] = { + "breeze testing tests": [ + { + "name": "Basic flag for tests command", + "options": [ + "--integration", + "--test-type", + "--db-reset", + "--backend", + "--python", + "--postgres-version", + "--mysql-version", + "--mssql-version", + ], + }, + { + "name": "Advanced flag for tests command", + "options": [ + "--limit-progress-output", + "--image-tag", + "--mount-sources", + ], + }, + ], + "breeze testing docker-compose-tests": [ + { + "name": "Docker-compose tests flag", + "options": [ + "--image-name", + "--image-tag", + "--python", + ], + } + ], +} diff --git a/dev/breeze/src/airflow_breeze/configure_rich_click.py b/dev/breeze/src/airflow_breeze/configure_rich_click.py index 8933d57dbeea3..71ea09c39a8a4 100644 --- a/dev/breeze/src/airflow_breeze/configure_rich_click.py +++ b/dev/breeze/src/airflow_breeze/configure_rich_click.py @@ -19,27 +19,34 @@ try: # We handle ImportError so that click autocomplete works import rich_click as click - - from airflow_breeze.commands.ci_commands import CI_COMMANDS, CI_PARAMETERS - from airflow_breeze.commands.ci_image_commands import CI_IMAGE_TOOLS_COMMANDS, CI_IMAGE_TOOLS_PARAMETERS - from airflow_breeze.commands.configuration_and_maintenance_commands import ( - CONFIGURATION_AND_MAINTENANCE_COMMANDS, - CONFIGURATION_AND_MAINTENANCE_PARAMETERS, +except ImportError as e: + if "No module named 'rich_click'" in e.msg: + # just ignore the import error when rich_click is missing + import click # type: ignore[no-redef] + else: + raise +else: + from airflow_breeze.commands.ci_commands_config import CI_COMMANDS, CI_PARAMETERS + from airflow_breeze.commands.ci_image_commands_config import ( + CI_IMAGE_TOOLS_COMMANDS, + CI_IMAGE_TOOLS_PARAMETERS, ) - from airflow_breeze.commands.developer_commands import DEVELOPER_COMMANDS, DEVELOPER_PARAMETERS - from airflow_breeze.commands.production_image_commands import ( + from airflow_breeze.commands.developer_commands_config import DEVELOPER_COMMANDS, DEVELOPER_PARAMETERS + from airflow_breeze.commands.production_image_commands_config import ( PRODUCTION_IMAGE_TOOLS_COMMANDS, PRODUCTION_IMAGE_TOOLS_PARAMETERS, ) - from airflow_breeze.commands.release_management_commands import ( + from airflow_breeze.commands.release_management_commands_config import ( RELEASE_MANAGEMENT_COMMANDS, RELEASE_MANAGEMENT_PARAMETERS, ) - from airflow_breeze.commands.testing_commands import TESTING_COMMANDS, TESTING_PARAMETERS + from airflow_breeze.commands.setup_commands_config import SETUP_COMMANDS, SETUP_PARAMETERS + from airflow_breeze.commands.testing_commands_config import TESTING_COMMANDS, TESTING_PARAMETERS click.rich_click.SHOW_METAVARS_COLUMN = False click.rich_click.SHOW_ARGUMENTS = False click.rich_click.APPEND_METAVARS_HELP = True + click.rich_click.OPTIONS_PANEL_TITLE = "Common options" click.rich_click.STYLE_ERRORS_SUGGESTION = "bright_blue italic" click.rich_click.ERRORS_SUGGESTION = "\nTry running the '--help' flag for more information.\n" click.rich_click.ERRORS_EPILOGUE = ( @@ -48,7 +55,7 @@ click.rich_click.OPTION_GROUPS = { **DEVELOPER_PARAMETERS, **TESTING_PARAMETERS, - **CONFIGURATION_AND_MAINTENANCE_PARAMETERS, + **SETUP_PARAMETERS, **CI_IMAGE_TOOLS_PARAMETERS, **PRODUCTION_IMAGE_TOOLS_PARAMETERS, **CI_PARAMETERS, @@ -57,17 +64,15 @@ click.rich_click.COMMAND_GROUPS = { "breeze": [ DEVELOPER_COMMANDS, - TESTING_COMMANDS, - CONFIGURATION_AND_MAINTENANCE_COMMANDS, - CI_IMAGE_TOOLS_COMMANDS, - PRODUCTION_IMAGE_TOOLS_COMMANDS, - CI_COMMANDS, - RELEASE_MANAGEMENT_COMMANDS, - ] + { + "name": "Advanced command groups", + "commands": ["testing", "ci-image", "prod-image", "setup", "release-management", "ci"], + }, + ], + "breeze testing": [TESTING_COMMANDS], + "breeze ci-image": [CI_IMAGE_TOOLS_COMMANDS], + "breeze prod-image": [PRODUCTION_IMAGE_TOOLS_COMMANDS], + "setup": [SETUP_COMMANDS], + "release-management": [RELEASE_MANAGEMENT_COMMANDS], + "breeze ci": [CI_COMMANDS], } -except ImportError as e: - if "No module named 'rich_click'" in e.msg: - # just ignore the import error when rich_click is missing - import click # type: ignore[no-redef] - else: - raise diff --git a/dev/breeze/src/airflow_breeze/params/common_build_params.py b/dev/breeze/src/airflow_breeze/params/common_build_params.py index 93cc8548a7fd8..320c9289c45f4 100644 --- a/dev/breeze/src/airflow_breeze/params/common_build_params.py +++ b/dev/breeze/src/airflow_breeze/params/common_build_params.py @@ -62,7 +62,7 @@ class CommonBuildParams: platform: str = DOCKER_DEFAULT_PLATFORM prepare_buildx_cache: bool = False python_image: Optional[str] = None - push_image: bool = False + push: bool = False python: str = "3.7" runtime_apt_command: str = "" runtime_apt_deps: str = "" diff --git a/dev/breeze/src/airflow_breeze/utils/click_utils.py b/dev/breeze/src/airflow_breeze/utils/click_utils.py new file mode 100644 index 0000000000000..4034593551541 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/utils/click_utils.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +try: + from rich_click import RichGroup as BreezeGroup +except ImportError: + from click import Group as BreezeGroup # type: ignore[misc] # noqa diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index ebfdb8e122892..ff91acf1d4fa7 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -308,11 +308,11 @@ is_flag=True, envvar='PREPARE_BUILDX_CACHE', ) -option_push_image = click.option( - '--push-image', +option_push = click.option( + '--push', help='Push image after building it.', is_flag=True, - envvar='PUSH_IMAGE', + envvar='PUSH', ) option_empty_image = click.option( '--empty-image', @@ -333,11 +333,11 @@ is_flag=True, envvar='TAG_AS_LATEST', ) -option_verify_image = click.option( - '--verify-image', +option_verify = click.option( + '--verify', help='Verify image.', is_flag=True, - envvar='VERIFY_IMAGE', + envvar='VERIFY', ) option_additional_pip_install_flags = click.option( '--additional-pip-install-flags', @@ -459,11 +459,11 @@ show_default=True, help='Mode of constraints for PROD image building', ) -option_pull_image = click.option( - '--pull-image', +option_pull = click.option( + '--pull', help="Pull image is missing before attempting to verify it.", is_flag=True, - envvar='PULL_IMAGE', + envvar='PULL', ) option_python_image = click.option( '--python-image', diff --git a/dev/breeze/src/airflow_breeze/utils/console.py b/dev/breeze/src/airflow_breeze/utils/console.py index 41ae65ef61158..18e6ca24376a6 100644 --- a/dev/breeze/src/airflow_breeze/utils/console.py +++ b/dev/breeze/src/airflow_breeze/utils/console.py @@ -79,3 +79,15 @@ def get_console() -> Console: theme=get_theme(), record=True if recording_file else False, ) + + +@lru_cache(maxsize=None) +def get_stderr_console() -> Console: + return Console( + force_terminal=True, + color_system="standard", + stderr=True, + width=180 if not recording_width else int(recording_width), + theme=get_theme(), + record=True if recording_file else False, + ) diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 5f0de41fbc3c4..164caf546cefe 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -433,7 +433,7 @@ def prepare_base_build_command(image_params: CommonBuildParams, verbose: bool) - "--builder", image_params.builder, "--progress=tty", - "--push" if image_params.push_image else "--load", + "--push" if image_params.push else "--load", ] ) else: @@ -688,7 +688,7 @@ def warm_up_docker_builder(image_params: CommonBuildParams, verbose: bool, dry_r get_console().print(f"[info]Warming up the {image_params.builder} builder for syntax: {docker_syntax}") warm_up_image_param = deepcopy(image_params) warm_up_image_param.image_tag = "warmup" - warm_up_image_param.push_image = False + warm_up_image_param.push = False build_command = prepare_base_build_command(image_params=warm_up_image_param, verbose=verbose) warm_up_command = [] warm_up_command.extend(["docker"]) diff --git a/dev/breeze/src/airflow_breeze/utils/image.py b/dev/breeze/src/airflow_breeze/utils/image.py index 57ccc2e3a129b..5d83e7fff5f13 100644 --- a/dev/breeze/src/airflow_breeze/utils/image.py +++ b/dev/breeze/src/airflow_breeze/utils/image.py @@ -43,7 +43,7 @@ def run_pull_in_parallel( image_params_list: Union[List[BuildCiParams], List[BuildProdParams]], python_version_list: List[str], verbose: bool, - verify_image: bool, + verify: bool, tag_as_latest: bool, wait_for_image: bool, extra_pytest_args: Tuple, @@ -54,7 +54,7 @@ def run_pull_in_parallel( ) pool = mp.Pool(parallelism) poll_time = 10.0 - if not verify_image: + if not verify: results = [ pool.apply_async( run_pull_image, diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py index c54b54e5c7f34..55aa5fd2e106e 100644 --- a/dev/breeze/src/airflow_breeze/utils/path_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py @@ -68,7 +68,7 @@ def skip_upgrade_check(): def get_package_setup_metadata_hash() -> str: """ - Retrieves hash of setup.py and setup.cfg files from the source of installation of Breeze. + Retrieves hash of setup files from the source of installation of Breeze. This is used in order to determine if we need to upgrade Breeze, because some setup files changed. Blake2b algorithm will not be flagged by security checkers @@ -117,7 +117,7 @@ def get_installation_sources_config_metadata_hash() -> str: def get_used_sources_setup_metadata_hash() -> str: """ - Retrieves hash of setup.py and setup.cfg files from the currently used sources. + Retrieves hash of setup files from the currently used sources. """ return get_sources_setup_metadata_hash(get_used_airflow_sources()) diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py b/dev/breeze/src/airflow_breeze/utils/run_utils.py index 4d99fa6a67d2e..ead77dc2daee2 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py @@ -187,7 +187,7 @@ def assert_pre_commit_installed(verbose: bool): else: get_console().print("\n[error]Error checking for pre-commit-installation:[/]\n") get_console().print(command_result.stderr) - get_console().print("\nMake sure to run:\n breeze self-upgrade\n\n") + get_console().print("\nMake sure to run:\n breeze setup self-upgrade\n\n") sys.exit(1) @@ -215,7 +215,8 @@ def instruct_build_image(python: str): """Print instructions to the user that they should build the image""" get_console().print(f'[warning]\nThe CI image for Python version {python} may be outdated[/]\n') get_console().print( - f"\n[info]Please run at the earliest convenience:[/]\n\nbreeze build-image --python {python}\n\n" + f"\n[info]Please run at the earliest " + f"convenience:[/]\n\nbreeze ci-image build --python {python}\n\n" ) @@ -351,7 +352,7 @@ def get_runnable_ci_image(verbose: bool, dry_run: bool) -> str: image=airflow_image, verbose=verbose, dry_run=dry_run, - instruction=f"breeze build-image --python {python_version}", + instruction=f"breeze ci-image build --python {python_version}", ) return airflow_image diff --git a/dev/breeze/src/airflow_breeze/utils/visuals.py b/dev/breeze/src/airflow_breeze/utils/visuals.py index b5db534514814..7702ed039018c 100644 --- a/dev/breeze/src/airflow_breeze/utils/visuals.py +++ b/dev/breeze/src/airflow_breeze/utils/visuals.py @@ -126,7 +126,7 @@ You can disable this cheatsheet by running: - ./breeze config --no-cheatsheet + ./breeze setup config --no-cheatsheet """ CHEATSHEET_STYLE = "white" diff --git a/dev/refresh_images.sh b/dev/refresh_images.sh index 7675db65b2206..6ae20ee24a7f2 100755 --- a/dev/refresh_images.sh +++ b/dev/refresh_images.sh @@ -23,9 +23,9 @@ export ANSWER="yes" export CI="true" export GITHUB_TOKEN="" -breeze self-upgrade --force --use-current-airflow-sources +breeze setup self-upgrade --force --use-current-airflow-sources -breeze build-image \ +breeze ci-image build \ --builder airflow_cache \ --run-in-parallel \ --prepare-buildx-cache \ @@ -35,16 +35,16 @@ breeze build-image \ rm -fv ./dist/* ./docker-context-files/* -breeze prepare-provider-packages \ +breeze release-management prepare-provider-packages \ --package-list-file ./scripts/ci/installed_providers.txt \ --package-format wheel \ --version-suffix-for-pypi dev0 -breeze prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0 +breeze release-management prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0 mv -v ./dist/*.whl ./docker-context-files -breeze build-prod-image \ +breeze prod-image build \ --builder airflow_cache \ --run-in-parallel \ --install-packages-from-context \ diff --git a/docker_tests/docker_tests_utils.py b/docker_tests/docker_tests_utils.py index 6b3512b3b572d..77960ebaebe44 100644 --- a/docker_tests/docker_tests_utils.py +++ b/docker_tests/docker_tests_utils.py @@ -83,11 +83,11 @@ def display_dependency_conflict_message(): CI image: - ./breeze build-image --upgrade-to-newer-dependencies --python 3.7 + ./breeze ci-image build --upgrade-to-newer-dependencies --python 3.7 Production image: - ./breeze build-image --production-image --upgrade-to-newer-dependencies --python 3.7 + ./breeze ci-image build --production-image --upgrade-to-newer-dependencies --python 3.7 * You will see error messages there telling which requirements are conflicting and which packages caused the conflict. Add the limitation that caused the conflict to EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS diff --git a/images/breeze/output-build-docs.svg b/images/breeze/output-build-docs.svg deleted file mode 100644 index 856e8b0595675..0000000000000 --- a/images/breeze/output-build-docs.svg +++ /dev/null @@ -1,296 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: build-docs - - - - - - - - - - -Usage: breeze build-docs [OPTIONS] - -Build documentation in the container. - -╭─ Doc flags ──────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--docs-only-dOnly build documentation.│ -│--spellcheck-only-sOnly run spell checking.│ -│--clean-buildClean inventories of Inter-Sphinx documentation and generated APIs and sphinx artifacts      │ -│before the build - useful for a clean build.                                                 │ -│--for-productionBuilds documentation for official release i.e. all links point to stable version. Implies    │ -│--clean-build│ -│--package-filterList of packages to consider.                                                                │ -│(apache-airflow | apache-airflow-providers | apache-airflow-providers-airbyte |              │ -│apache-airflow-providers-alibaba | apache-airflow-providers-amazon |                         │ -│apache-airflow-providers-apache-beam | apache-airflow-providers-apache-cassandra |           │ -│apache-airflow-providers-apache-drill | apache-airflow-providers-apache-druid |              │ -│apache-airflow-providers-apache-hdfs | apache-airflow-providers-apache-hive |                │ -│apache-airflow-providers-apache-kylin | apache-airflow-providers-apache-livy |               │ -│apache-airflow-providers-apache-pig | apache-airflow-providers-apache-pinot |                │ -│apache-airflow-providers-apache-spark | apache-airflow-providers-apache-sqoop |              │ -│apache-airflow-providers-arangodb | apache-airflow-providers-asana |                         │ -│apache-airflow-providers-celery | apache-airflow-providers-cloudant |                        │ -│apache-airflow-providers-cncf-kubernetes | apache-airflow-providers-common-sql |             │ -│apache-airflow-providers-databricks | apache-airflow-providers-datadog |                     │ -│apache-airflow-providers-dbt-cloud | apache-airflow-providers-dingding |                     │ -│apache-airflow-providers-discord | apache-airflow-providers-docker |                         │ -│apache-airflow-providers-elasticsearch | apache-airflow-providers-exasol |                   │ -│apache-airflow-providers-facebook | apache-airflow-providers-ftp |                           │ -│apache-airflow-providers-github | apache-airflow-providers-google |                          │ -│apache-airflow-providers-grpc | apache-airflow-providers-hashicorp |                         │ -│apache-airflow-providers-http | apache-airflow-providers-imap |                              │ -│apache-airflow-providers-influxdb | apache-airflow-providers-jdbc |                          │ -│apache-airflow-providers-jenkins | apache-airflow-providers-jira |                           │ -│apache-airflow-providers-microsoft-azure | apache-airflow-providers-microsoft-mssql |        │ -│apache-airflow-providers-microsoft-psrp | apache-airflow-providers-microsoft-winrm |         │ -│apache-airflow-providers-mongo | apache-airflow-providers-mysql |                            │ -│apache-airflow-providers-neo4j | apache-airflow-providers-odbc |                             │ -│apache-airflow-providers-openfaas | apache-airflow-providers-opsgenie |                      │ -│apache-airflow-providers-oracle | apache-airflow-providers-pagerduty |                       │ -│apache-airflow-providers-papermill | apache-airflow-providers-plexus |                       │ -│apache-airflow-providers-postgres | apache-airflow-providers-presto |                        │ -│apache-airflow-providers-qubole | apache-airflow-providers-redis |                           │ -│apache-airflow-providers-salesforce | apache-airflow-providers-samba |                       │ -│apache-airflow-providers-segment | apache-airflow-providers-sendgrid |                       │ -│apache-airflow-providers-sftp | apache-airflow-providers-singularity |                       │ -│apache-airflow-providers-slack | apache-airflow-providers-snowflake |                        │ -│apache-airflow-providers-sqlite | apache-airflow-providers-ssh |                             │ -│apache-airflow-providers-tableau | apache-airflow-providers-tabular |                        │ -│apache-airflow-providers-telegram | apache-airflow-providers-trino |                         │ -│apache-airflow-providers-vertica | apache-airflow-providers-yandex |                         │ -│apache-airflow-providers-zendesk | docker-stack | helm-chart)                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-build-image.svg b/images/breeze/output-build-image.svg deleted file mode 100644 index 3176d38205390..0000000000000 --- a/images/breeze/output-build-image.svg +++ /dev/null @@ -1,332 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: build-image - - - - - - - - - - -Usage: breeze build-image [OPTIONS] - -Build CI image. Include building multiple images for all python versions (sequentially). - -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ -│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ -│--image-tag-tTag the image after building it(TEXT)│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ -│when you build or pull image with --image-tag.                                 │ -│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ -│[default: registry]                          │ -│--force-buildForce image build no matter if it is determined as needed.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ -│--install-providers-from-sourcesInstall providers from sources when installing.│ -│--airflow-constraints-modeMode of constraints for CI image building                              │ -│(constraints-source-providers | constraints | constraints-no-providers)│ -│[default: constraints-source-providers]                                │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ -│--python-imageIf specified this is the base python image used to build the image. Should be   │ -│something like: python:VERSION-slim-bullseye                                    │ -│(TEXT)                                                                          │ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ -│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ -│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ -│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ -│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--push-imagePush image after building it.│ -│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ -│image).                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ -│[default: apache/airflow]                       │ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--builderBuildx builder used to perform `docker buildx build` commands(TEXT)│ -│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ -│itself).                                                                        │ -│(TEXT)                                                                          │ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-build-prod-image.svg b/images/breeze/output-build-prod-image.svg deleted file mode 100644 index 32e4d19ef43d7..0000000000000 --- a/images/breeze/output-build-prod-image.svg +++ /dev/null @@ -1,396 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: build-prod-image - - - - - - - - - - -Usage: breeze build-prod-image [OPTIONS] - -Build Production image. Include building multiple images for all or selected Python versions sequentially. - -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ -│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ -│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ -│--image-tag-tTag the image after building it(TEXT)│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ -│when you build or pull image with --image-tag.                                 │ -│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ -│[default: registry]                          │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ -│--install-providers-from-sourcesInstall providers from sources when installing.│ -│--airflow-extrasExtras to install by default.                                                   │ -│(TEXT)                                                                          │ -│[default:                                                                       │ -│amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google…│ -│--airflow-constraints-modeMode of constraints for PROD image building                            │ -│(constraints | constraints-no-providers | constraints-source-providers)│ -│[default: constraints]                                                 │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ -│--python-imageIf specified this is the base python image used to build the image. Should be   │ -│something like: python:VERSION-slim-bullseye                                    │ -│(TEXT)                                                                          │ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ -│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ -│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ -│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ -│--install-packages-from-contextInstall wheels from local docker-context-files when building image.       │ -│Implies --disable-airflow-repo-cache.                                     │ -│--cleanup-contextClean up docker context files before running build (cannot be used        │ -│together with --install-packages-from-context).                           │ -│--disable-mysql-client-installationDo not install MySQL client.│ -│--disable-mssql-client-installationDo not install MsSQl client.│ -│--disable-postgres-client-installationDo not install Postgres client.│ -│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ -│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ -│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--push-imagePush image after building it.│ -│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ -│image).                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ -│[default: apache/airflow]                       │ -│--builderBuildx builder used to perform `docker buildx build` commands(TEXT)│ -│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ -│itself).                                                                        │ -│(TEXT)                                                                          │ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-cleanup.svg b/images/breeze/output-cleanup.svg deleted file mode 100644 index 9bd373b98be83..0000000000000 --- a/images/breeze/output-cleanup.svg +++ /dev/null @@ -1,120 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: cleanup - - - - - - - - - - -Usage: breeze cleanup [OPTIONS] - -Cleans the cache of parameters, docker cache and optionally - currently downloaded images. - -╭─ Cleanup flags ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--allAlso remove currently downloaded Breeze images.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-command-hash-export.svg b/images/breeze/output-command-hash-export.svg deleted file mode 100644 index 789317d72494f..0000000000000 --- a/images/breeze/output-command-hash-export.svg +++ /dev/null @@ -1,95 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: command-hash-export - - - - - - - - - - -Usage: breeze command-hash-export [OPTIONS] OUTPUT - -Outputs hash of all click commands to file or stdout if `-` is used (useful to see if images should be regenerated). - -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index c9ac2d5a594b1..c3e07dea0a7ea 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -1,38 +1,67 @@ # This file is automatically generated by pre-commit. If you have a conflict with this file -# Please do not solve it but run `breeze regenerate-command-images`. +# Please do not solve it but run `breeze setup regenerate-command-images`. # This command should fix the conflict and regenerate help images that you have conflict with. -main:fa4319079b275ce966502346f083f2e3 +main:b9b784d13daf2ab79e4242cdba476018 build-docs:74c301f05bbd185a19fd185f1873e38a -build-image:33d327fda1d00c35407ef8c00eea8aca -build-prod-image:db497d0a0f34392211c31ab9e0e8e7b8 -cleanup:9a94bd1063296ea86e895f671db0b330 -command-hash-export:83bc6a4a8c60b62da3d0f00e81d2c3ea +ci:chain:68934a3e9455fa72420237eb05902327 +ci:commands:894e9c942da72a80a67769ad48a74d06 +ci:deprecated:68934a3e9455fa72420237eb05902327 +ci:epilog:37a6259cc0c1dae299a7866489dff0bd +ci:help:e2dc4bf71f1ab9683c0848693f4470ff +ci:hidden:68934a3e9455fa72420237eb05902327 +ci:name:acfd600dd00e51cb9aee63a773ea796f +ci:params:bc6b726d05240c22ac8522edc4a200ee +ci:short_help:37a6259cc0c1dae299a7866489dff0bd +ci-image:chain:68934a3e9455fa72420237eb05902327 +ci-image:commands:bd889b1fd072636faad779e495019a6c +ci-image:deprecated:68934a3e9455fa72420237eb05902327 +ci-image:epilog:37a6259cc0c1dae299a7866489dff0bd +ci-image:help:c72b2b4bb402fe4168d23dacd879cf68 +ci-image:hidden:68934a3e9455fa72420237eb05902327 +ci-image:name:b05efacaf44e34e2023a9619df0a868c +ci-image:params:bc6b726d05240c22ac8522edc4a200ee +ci-image:short_help:37a6259cc0c1dae299a7866489dff0bd +cleanup:9bf46a1dfd9db4fe13a1c233ad1bb96b compile-www-assets:23675c1862d0968cbff6ab6f1d93d488 -config:92653afc11889e1b78e3a2e38f41107f -docker-compose-tests:8ae3b6211fd31db81a750d1c6b96ec3d -exec:e4329909b8b2a610fa4fad5116c4b896 -find-newer-dependencies:00000f7afb289e36e8c573fcc654df44 -fix-ownership:84902165a54467564fbdd3598fa273e2 -free-space:bb8e7ac63d12ab3ede272a898de2f527 -generate-constraints:a5120e79439f30eb7fbee929dca23156 -prepare-airflow-package:cff9d88ca313db10f3cc464c6798f6be -prepare-provider-documentation:ff90e2d37c629e0f7b1f5e8bc723d9db -prepare-provider-packages:349292885c763f32db2bb8f99ae0ae59 -pull-image:a9bb83372b5da5212f48e2affeedc551 -pull-prod-image:6e8467a2b8c833a392c8bdd65189363e -regenerate-command-images:4fd2e7ecbfd6eebb18b854f3eb0f29c8 -release-prod-images:8858fe5a13989c7c65a79dc97a880928 -resource-check:0fb929ac3496dbbe97acfe99e35accd7 -selective-check:d4e3c250cd6f2b0040fbe6557fa423f6 -self-upgrade:b5437c0a1a91533a11ee9d0a9692369c -setup-autocomplete:355b72dee171c2fcba46fc90ac7c97b0 -shell:4680295fdd8a276d51518d29360c365c -start-airflow:92cf775a952439a32d409cd2536da507 +exec:89b81bc34d45b0fe6653a6db5482258c +prod-image:chain:68934a3e9455fa72420237eb05902327 +prod-image:commands:9b60954b229d7c6720958fbf53c1d61e +prod-image:deprecated:68934a3e9455fa72420237eb05902327 +prod-image:epilog:37a6259cc0c1dae299a7866489dff0bd +prod-image:help:57b6db594b341bc5d22a1195db4fbae9 +prod-image:hidden:68934a3e9455fa72420237eb05902327 +prod-image:name:8b97e28b5f9256845909c677a9187216 +prod-image:params:bc6b726d05240c22ac8522edc4a200ee +prod-image:short_help:37a6259cc0c1dae299a7866489dff0bd +release-management:chain:68934a3e9455fa72420237eb05902327 +release-management:commands:670c1b7051c96536eede3b3116f1d090 +release-management:deprecated:68934a3e9455fa72420237eb05902327 +release-management:epilog:37a6259cc0c1dae299a7866489dff0bd +release-management:help:b1ca9edf1caadef8f13fc806c1261332 +release-management:hidden:68934a3e9455fa72420237eb05902327 +release-management:name:23bcbf95c4ca34de89a367669f7274c1 +release-management:params:bc6b726d05240c22ac8522edc4a200ee +release-management:short_help:37a6259cc0c1dae299a7866489dff0bd +setup:chain:68934a3e9455fa72420237eb05902327 +setup:commands:c95a46d890baef83220804e613c55542 +setup:deprecated:68934a3e9455fa72420237eb05902327 +setup:epilog:37a6259cc0c1dae299a7866489dff0bd +setup:help:53e8d17d2f9a677f11cc08495a7a5012 +setup:hidden:68934a3e9455fa72420237eb05902327 +setup:name:2e5fb4b1fd75f804556bdf9c033f9086 +setup:params:bc6b726d05240c22ac8522edc4a200ee +setup:short_help:37a6259cc0c1dae299a7866489dff0bd +shell:7baeb98c0103c594502ccf3d32ae24c6 +start-airflow:3e793b11dc2158c54bfc189bfe20d6f2 static-checks:03e96245e60c225ed94514ad7b42ceb0 stop:8ebd8a42f1003495d37b884de5ac7ce6 -tests:e39111ecbd33a65ababb3cbfbac2b069 -verify-image:a6b3c70957aea96a5d4d261f23359a2d -verify-prod-image:bf3cf39200e010e3015ef071fd387c6f -verify-provider-packages:797e60067fc4611112527de808b5c1c1 -version:d11da4c17a23179830079b646160149c +testing:chain:68934a3e9455fa72420237eb05902327 +testing:commands:52beb8911b727d83e048cec69ba1f4a6 +testing:deprecated:68934a3e9455fa72420237eb05902327 +testing:epilog:37a6259cc0c1dae299a7866489dff0bd +testing:help:590a1f3d30c4015a1df4acb1a622c66d +testing:hidden:68934a3e9455fa72420237eb05902327 +testing:name:fa0f2235fd10b78d61fbcc2177c5b2b8 +testing:params:bc6b726d05240c22ac8522edc4a200ee +testing:short_help:37a6259cc0c1dae299a7866489dff0bd diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index 77e487bfd108f..92b328dfecc61 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Breeze commands + Breeze commands - + - - -Usage: breeze [OPTIONS] COMMAND [ARGS]... - -╭─ Basic flags for the default (shell) command ────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ -│all)                                                                                     │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flags for the default (shell) command ─────────────────────────────────────────────────────────────────────╮ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or  │ -│`sdist` if Airflow should be removed, installed from wheel packages or sdist packages   │ -│available in dist folder respectively. Implies --mount-sources `remove`.                │ -│(none | wheel | sdist | <airflow_version>)                                              │ -│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ -│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist' folder when   │ -│entering breeze.                                                                        │ -│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)[default: wheel]│ -│--force-buildForce image build no matter if it is determined as needed.│ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =    │ -│selected).                                                                              │ -│(selected | all | skip | remove)                                                        │ -│[default: selected]                                                                     │ -│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Developer tools ────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│shell               Enter breeze.py environment. this is the default command use when no other is selected.       │ -│start-airflow       Enter breeze.py environment and starts all Airflow components in the tmux session.            │ -│compile-www-assets  Compiles www assets.                                                                          │ -│exec                Joins the interactive shell of running airflow container                                      │ -│stop                Stop running breeze environment.                                                              │ -│build-docs          Build documentation in the container.                                                         │ -│static-checks       Run static checks.                                                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│docker-compose-tests Run docker-compose tests.                                                                    │ -│tests                Run the specified unit test targets. Multiple targets may be specified separated by spaces.  │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Configuration & maintenance ────────────────────────────────────────────────────────────────────────────────────────╮ -│cleanup                  Cleans the cache of parameters, docker cache and optionally - currently downloaded       │ -│images.                                                                                  │ -│self-upgrade             Self upgrade Breeze.                                                                     │ -│setup-autocomplete       Enables autocompletion of breeze commands.                                               │ -│config                   Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).                       │ -│regenerate-command-imagesRegenerate breeze command images.                                                        │ -│command-hash-export      Outputs hash of all click commands to file or stdout if `-` is used (useful to see if    │ -│images should be regenerated).                                                           │ -│version                  Print information about version of apache-airflow-breeze.                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ CI Image tools ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│build-image   Build CI image. Include building multiple images for all python versions (sequentially).            │ -│pull-image    Pull and optionally verify CI images - possibly in parallel for all Python versions.                │ -│verify-image  Verify CI image.                                                                                    │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Production Image tools ─────────────────────────────────────────────────────────────────────────────────────────────╮ -│build-prod-image Build Production image. Include building multiple images for all or selected Python versions     │ -│sequentially.                                                                                    │ -│pull-prod-image  Pull and optionally verify Production images - possibly in parallel for all Python versions.     │ -│verify-prod-imageVerify Production image.                                                                         │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ CI commands ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│fix-ownership                  Fix ownership of source files to be same as host user.                             │ -│free-space                     Free space for jobs run in CI.                                                     │ -│resource-check                 Check if available docker resources are enough.                                    │ -│selective-check                Checks what kind of tests should be run for an incoming commit.                    │ -│find-newer-dependencies        Finds which dependencies are being upgraded.                                       │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Release management ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│verify-provider-packages         Verifies if all provider code is following expectations for providers.           │ -│prepare-provider-documentation   Prepare CHANGELOG, README and COMMITS information for providers.                 │ -│prepare-provider-packages        Prepare sdist/whl packages of Airflow Providers.                                 │ -│prepare-airflow-package          Prepare sdist/whl package of Airflow.                                            │ -│release-prod-images              Release production images to DockerHub (needs DockerHub permissions).            │ -│generate-constraints             Generates pinned constraint files with all extras from setup.py in parallel.     │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze [OPTIONS] COMMAND [ARGS]... + +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                            │ +│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ +│all)                                                                                     │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic developer commands ───────────────────────────────────────────────────────────────────────────────────────────╮ +│start-airflow        Enter breeze environment and starts all Airflow components in the tmux session.              │ +│static-checks        Run static checks.                                                                           │ +│build-docs           Build documentation in the container.                                                        │ +│stop                 Stop running breeze environment.                                                             │ +│shell                Enter breeze environment. this is the default command use when no other is selected.         │ +│exec                 Joins the interactive shell of running airflow container.                                    │ +│compile-www-assets   Compiles www assets.                                                                         │ +│cleanup              Cleans the cache of parameters, docker cache and optionally built CI/PROD images.            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced command groups ────────────────────────────────────────────────────────────────────────────────────────────╮ +│testing                Tools that developers can use to run tests                                                 │ +│ci-image               Tools that developers can use to manually manage CI images                                 │ +│prod-image             Tools that developers can use to manually manage PROD images                               │ +│setup                  Tools that developers can use to configure Breeze                                          │ +│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ +│ci                     Tools that CI workflows use to cleanup/manage CI environment                               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-compile-www-assets.svg b/images/breeze/output-compile-www-assets.svg deleted file mode 100644 index da06d16782d49..0000000000000 --- a/images/breeze/output-compile-www-assets.svg +++ /dev/null @@ -1,115 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: compile-www-assets - - - - - - - - - - -Usage: breeze compile-www-assets [OPTIONS] - -Compiles www assets. - -╭─ Compile www assets flag ────────────────────────────────────────────────────────────────────────────────────────────╮ -│--devRun development version of assets compilation - it will not quit and automatically recompile assets        │ -│on-the-fly when they are changed.                                                                          │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-config.svg b/images/breeze/output-config.svg deleted file mode 100644 index 4f161b859fc3e..0000000000000 --- a/images/breeze/output-config.svg +++ /dev/null @@ -1,144 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: config - - - - - - - - - - -Usage: breeze config [OPTIONS] - -Show/update configuration (Python, Backend, Cheatsheet, ASCIIART). - -╭─ Config flags ───────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)│ -│[default: sqlite]       │ -│--cheatsheet/--no-cheatsheet-C/-cEnable/disable cheatsheet.│ -│--asciiart/--no-asciiart-A/-aEnable/disable ASCIIart.│ -│--colour/--no-colourEnable/disable Colour mode (useful for colour blind-friendly communication).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-docker-compose-tests.svg b/images/breeze/output-docker-compose-tests.svg deleted file mode 100644 index 424d3cda29fc8..0000000000000 --- a/images/breeze/output-docker-compose-tests.svg +++ /dev/null @@ -1,128 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: docker-compose-tests - - - - - - - - - - -Usage: breeze docker-compose-tests [OPTIONS] [EXTRA_PYTEST_ARGS]... - -Run docker-compose tests. - -╭─ Docker-compose tests flag ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip)(TEXT)│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-exec.svg b/images/breeze/output-exec.svg deleted file mode 100644 index b8e69cd3addef..0000000000000 --- a/images/breeze/output-exec.svg +++ /dev/null @@ -1,99 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: exec - - - - - - - - - - -Usage: breeze exec [OPTIONS] [EXEC_ARGS]... - -Joins the interactive shell of running airflow container - -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-find-newer-dependencies.svg b/images/breeze/output-find-newer-dependencies.svg deleted file mode 100644 index 43fc632ead7ed..0000000000000 --- a/images/breeze/output-find-newer-dependencies.svg +++ /dev/null @@ -1,132 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: find-newer-dependencies - - - - - - - - - - -Usage: breeze find-newer-dependencies [OPTIONS] - -Finds which dependencies are being upgraded. - -╭─ Find newer dependencies flags ──────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--timezoneTimezone to use during the check(TEXT)│ -│--updated-on-or-afterDate when the release was updated after(TEXT)│ -│--max-ageMax age of the last release (used if no updated-on-or-after if specified)(INTEGER)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ -│specify constraints for the installed version and to find newer dependencies   │ -│(TEXT)                                                                         │ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-fix-ownership.svg b/images/breeze/output-fix-ownership.svg deleted file mode 100644 index f7208613b1a6c..0000000000000 --- a/images/breeze/output-fix-ownership.svg +++ /dev/null @@ -1,116 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: fix-ownership - - - - - - - - - - -Usage: breeze fix-ownership [OPTIONS] - -Fix ownership of source files to be same as host user. - -╭─ Fix ownership flags ────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--use-sudoUse sudo instead of docker image to fix the ownership. You need to be a `sudoer` to run it│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-free-space.svg b/images/breeze/output-free-space.svg deleted file mode 100644 index e0e4296201385..0000000000000 --- a/images/breeze/output-free-space.svg +++ /dev/null @@ -1,104 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: free-space - - - - - - - - - - -Usage: breeze free-space [OPTIONS] - -Free space for jobs run in CI. - -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-generate-constraints.svg b/images/breeze/output-generate-constraints.svg deleted file mode 100644 index 24663d4bc7aea..0000000000000 --- a/images/breeze/output-generate-constraints.svg +++ /dev/null @@ -1,180 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: generate-constraints - - - - - - - - - - -Usage: breeze generate-constraints [OPTIONS] - -Generates pinned constraint files with all extras from setup.py in parallel. - -╭─ Generate constraints flags ─────────────────────────────────────────────────────────────────────────────────────────╮ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip)│ -│(TEXT)                                                                        │ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--airflow-constraints-modeMode of constraints for CI image building                              │ -│(constraints-source-providers | constraints | constraints-no-providers)│ -│[default: constraints-source-providers]                                │ -│--debugDrop user in shell instead of running the command. Useful for debugging.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-prepare-airflow-package.svg b/images/breeze/output-prepare-airflow-package.svg deleted file mode 100644 index 996151f94cc6d..0000000000000 --- a/images/breeze/output-prepare-airflow-package.svg +++ /dev/null @@ -1,124 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: prepare-airflow-package - - - - - - - - - - -Usage: breeze prepare-airflow-package [OPTIONS] - -Prepare sdist/whl package of Airflow. - -╭─ Package flags ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--package-formatFormat of packages.(wheel | sdist | both)[default: wheel]│ -│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--debugDrop user in shell instead of running the command. Useful for debugging.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-prepare-provider-documentation.svg b/images/breeze/output-prepare-provider-documentation.svg deleted file mode 100644 index c413c49f8c6d7..0000000000000 --- a/images/breeze/output-prepare-provider-documentation.svg +++ /dev/null @@ -1,168 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: prepare-provider-documentation - - - - - - - - - - -Usage: breeze prepare-provider-documentation [OPTIONS] [airbyte | alibaba | amazon | apache.beam | apache.cassandra | -                                             apache.drill | apache.druid | apache.hdfs | apache.hive | apache.kylin | -                                             apache.livy | apache.pig | apache.pinot | apache.spark | apache.sqoop | -                                             arangodb | asana | celery | cloudant | cncf.kubernetes | common.sql | -                                             databricks | datadog | dbt.cloud | dingding | discord | docker | -                                             elasticsearch | exasol | facebook | ftp | github | google | grpc | -                                             hashicorp | http | imap | influxdb | jdbc | jenkins | jira | -                                             microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | -                                             mongo | mysql | neo4j | odbc | openfaas | opsgenie | oracle | pagerduty | -                                             papermill | plexus | postgres | presto | qubole | redis | salesforce | -                                             samba | segment | sendgrid | sftp | singularity | slack | snowflake | -                                             sqlite | ssh | tableau | tabular | telegram | trino | vertica | yandex | -                                             zendesk]... - -Prepare CHANGELOG, README and COMMITS information for providers. - -╭─ Provider documentation preparation flags ───────────────────────────────────────────────────────────────────────────╮ -│--debugDrop user in shell instead of running the command. Useful for debugging.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-prepare-provider-packages.svg b/images/breeze/output-prepare-provider-packages.svg deleted file mode 100644 index aa4ba89f53582..0000000000000 --- a/images/breeze/output-prepare-provider-packages.svg +++ /dev/null @@ -1,172 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: prepare-provider-packages - - - - - - - - - - -Usage: breeze prepare-provider-packages [OPTIONS] [airbyte | alibaba | amazon | apache.beam | apache.cassandra | -                                        apache.drill | apache.druid | apache.hdfs | apache.hive | apache.kylin | -                                        apache.livy | apache.pig | apache.pinot | apache.spark | apache.sqoop | -                                        arangodb | asana | celery | cloudant | cncf.kubernetes | common.sql | -                                        databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch -                                        | exasol | facebook | ftp | github | google | grpc | hashicorp | http | imap | -                                        influxdb | jdbc | jenkins | jira | microsoft.azure | microsoft.mssql | -                                        microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openfaas | -                                        opsgenie | oracle | pagerduty | papermill | plexus | postgres | presto | -                                        qubole | redis | salesforce | samba | segment | sendgrid | sftp | singularity -                                        | slack | snowflake | sqlite | ssh | tableau | tabular | telegram | trino | -                                        vertica | yandex | zendesk]... - -Prepare sdist/whl packages of Airflow Providers. - -╭─ Package flags ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--package-formatFormat of packages.(wheel | sdist | both)[default: wheel]│ -│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ -│--package-list-fileRead list of packages from text file (one package per line)(FILENAME)│ -│--debugDrop user in shell instead of running the command. Useful for debugging.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-pull-image.svg b/images/breeze/output-pull-image.svg deleted file mode 100644 index 7ebae02e45d41..0000000000000 --- a/images/breeze/output-pull-image.svg +++ /dev/null @@ -1,174 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: pull-image - - - - - - - - - - -Usage: breeze pull-image [OPTIONS] [EXTRA_PYTEST_ARGS]... - -Pull and optionally verify CI images - possibly in parallel for all Python versions. - -╭─ Pull image flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│*--image-tag-tTag of the image which is used to pull the image(TEXT)[required]│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--verify-imageVerify image.│ -│--wait-for-imageWait until image is available.│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when you   │ -│build or pull image with --image-tag.                                                      │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-pull-prod-image.svg b/images/breeze/output-pull-prod-image.svg deleted file mode 100644 index 6cb1051d80c49..0000000000000 --- a/images/breeze/output-pull-prod-image.svg +++ /dev/null @@ -1,174 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: pull-prod-image - - - - - - - - - - -Usage: breeze pull-prod-image [OPTIONS] [EXTRA_PYTEST_ARGS]... - -Pull and optionally verify Production images - possibly in parallel for all Python versions. - -╭─ Pull image flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│*--image-tag-tTag of the image which is used to pull the image(TEXT)[required]│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--verify-imageVerify image.│ -│--wait-for-imageWait until image is available.│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when you   │ -│build or pull image with --image-tag.                                                      │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-regenerate-command-images.svg b/images/breeze/output-regenerate-command-images.svg deleted file mode 100644 index 0842e64fa1da3..0000000000000 --- a/images/breeze/output-regenerate-command-images.svg +++ /dev/null @@ -1,99 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: regenerate-command-images - - - - - - - - - - -Usage: breeze regenerate-command-images [OPTIONS] - -Regenerate breeze command images. - -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-release-prod-images.svg b/images/breeze/output-release-prod-images.svg deleted file mode 100644 index a3a392acc5583..0000000000000 --- a/images/breeze/output-release-prod-images.svg +++ /dev/null @@ -1,158 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: release-prod-images - - - - - - - - - - -Usage: breeze release-prod-images [OPTIONS] - -Release production images to DockerHub (needs DockerHub permissions). - -╭─ Release PROD IMAGE flags ───────────────────────────────────────────────────────────────────────────────────────────╮ -│*--airflow-versionAirflow version to release (2.3.0, 2.3.0rc1 etc.)(TEXT)[required]│ -│--dockerhub-repoDockerHub repository for the images(TEXT)[default: apache/airflow]│ -│--slim-imagesWhether to prepare slim images instead of the regular ones.│ -│--limit-pythonSpecific python to build slim images for (if not specified - the images are built for all   │ -│available python versions)                                                                  │ -│(3.7 | 3.8 | 3.9 | 3.10)                                                                    │ -│--limit-platformSpecific platform to build images for (if not specified, multiplatform images will be built.│ -│(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)                                       │ -│[default: linux/amd64,linux/arm64]                                                          │ -│--skip-latestWhether to skip publishing the latest images (so that 'latest' images are not updated). This│ -│should only be used if you release image for previous branches. Automatically set when      │ -│rc/alpha/beta images are built.                                                             │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-resource-check.svg b/images/breeze/output-resource-check.svg deleted file mode 100644 index 8ec8aa32d7e7c..0000000000000 --- a/images/breeze/output-resource-check.svg +++ /dev/null @@ -1,99 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: resource-check - - - - - - - - - - -Usage: breeze resource-check [OPTIONS] - -Check if available docker resources are enough. - -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-selective-check.svg b/images/breeze/output-selective-check.svg deleted file mode 100644 index 966079401a655..0000000000000 --- a/images/breeze/output-selective-check.svg +++ /dev/null @@ -1,144 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: selective-check - - - - - - - - - - -Usage: breeze selective-check [OPTIONS] - -Checks what kind of tests should be run for an incoming commit. - -╭─ Selective check flags ──────────────────────────────────────────────────────────────────────────────────────────────╮ -│--commit-refCommit-ish reference to the commit that should be checked(TEXT)│ -│--pr-labelsPython array formatted PR labels assigned to the PR(TEXT)│ -│--default-branchBranch against which the PR should be run(TEXT)[default: main]│ -│--github-event-nameName of the GitHub event that triggered the check                                            │ -│(pull_request | pull_request_review | pull_request_target | pull_request_workflow | push |   │ -│schedule | workflow_dispatch | workflow_run)                                                 │ -│[default: pull_request]                                                                      │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--default-constraints-branchConstraints Branch against which the PR should be run(TEXT)│ -│[default: constraints-main]                          │ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-self-upgrade.svg b/images/breeze/output-self-upgrade.svg deleted file mode 100644 index 205b6f1042e64..0000000000000 --- a/images/breeze/output-self-upgrade.svg +++ /dev/null @@ -1,107 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: self-upgrade - - - - - - - - - - -Usage: breeze self-upgrade [OPTIONS] - -Self upgrade Breeze. - -╭─ Self-upgrade flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--use-current-airflow-sources-aUse current workdir Airflow sources for upgrade.│ -│--force-fForce upgrade without asking question to the user.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-setup-autocomplete.svg b/images/breeze/output-setup-autocomplete.svg deleted file mode 100644 index de7c17ab43ee5..0000000000000 --- a/images/breeze/output-setup-autocomplete.svg +++ /dev/null @@ -1,116 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: setup-autocomplete - - - - - - - - - - -Usage: breeze setup-autocomplete [OPTIONS] - -Enables autocompletion of breeze commands. - -╭─ Setup autocomplete flags ───────────────────────────────────────────────────────────────────────────────────────────╮ -│--force-fForce autocomplete setup even if already setup before (overrides the setup).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-shell.svg b/images/breeze/output-shell.svg deleted file mode 100644 index 86ab4cc4085e5..0000000000000 --- a/images/breeze/output-shell.svg +++ /dev/null @@ -1,252 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: shell - - - - - - - - - - -Usage: breeze shell [OPTIONS] [EXTRA_ARGS]... - -Enter breeze.py environment. this is the default command use when no other is selected. - -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ -│all)                                                                                     │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or│ -│`sdist` if Airflow should be removed, installed from wheel packages or sdist packages │ -│available in dist folder respectively. Implies --mount-sources `remove`.              │ -│(none | wheel | sdist | <airflow_version>)                                            │ -│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ -│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist' folder when │ -│entering breeze.                                                                      │ -│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ -│[default: wheel]                                      │ -│--force-buildForce image build no matter if it is determined as needed.│ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip)(TEXT)│ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =  │ -│selected).                                                                            │ -│(selected | all | skip | remove)                                                      │ -│[default: selected]                                                                   │ -│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ -│[default: apache/airflow]                       │ -│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ -│specify constraints for the installed version and to find newer dependencies   │ -│(TEXT)                                                                         │ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-start-airflow.svg b/images/breeze/output-start-airflow.svg deleted file mode 100644 index 37031e3733de5..0000000000000 --- a/images/breeze/output-start-airflow.svg +++ /dev/null @@ -1,260 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: start-airflow - - - - - - - - - - -Usage: breeze start-airflow [OPTIONS] [EXTRA_ARGS]... - -Enter breeze.py environment and starts all Airflow components in the tmux session. - -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--load-example-dags-eEnable configuration to load example DAGs when starting Airflow.│ -│--load-default-connections-cEnable configuration to load default connections when starting Airflow.│ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                       │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino│ -│| all)                                                                              │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or│ -│`sdist` if Airflow should be removed, installed from wheel packages or sdist packages │ -│available in dist folder respectively. Implies --mount-sources `remove`.              │ -│(none | wheel | sdist | <airflow_version>)                                            │ -│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ -│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist' folder when │ -│entering breeze.                                                                      │ -│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ -│[default: wheel]                                      │ -│--force-buildForce image build no matter if it is determined as needed.│ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip)(TEXT)│ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =  │ -│selected).                                                                            │ -│(selected | all | skip | remove)                                                      │ -│[default: selected]                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ -│[default: apache/airflow]                       │ -│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ -│specify constraints for the installed version and to find newer dependencies   │ -│(TEXT)                                                                         │ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-static-checks.svg b/images/breeze/output-static-checks.svg deleted file mode 100644 index 37c6deb229b3e..0000000000000 --- a/images/breeze/output-static-checks.svg +++ /dev/null @@ -1,264 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: static-checks - - - - - - - - - - -Usage: breeze static-checks [OPTIONS] [PRECOMMIT_ARGS]... - -Run static checks. - -╭─ Pre-commit flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--type-tType(s) of the static checks to run (multiple can be added).                            │ -│(all | black | blacken-docs | check-airflow-2-2-compatibility |                         │ -│check-airflow-config-yaml-consistent | check-apache-license-rat |                       │ -│check-base-operator-partial-arguments | check-base-operator-usage |                     │ -│check-boring-cyborg-configuration | check-breeze-top-dependencies-limited |             │ -│check-builtin-literals | check-changelog-has-no-duplicates |                            │ -│check-daysago-import-from-utils | check-docstring-param-types | check-example-dags-urls │ -│| check-executables-have-shebangs | check-extra-packages-references | check-extras-order│ -│| check-for-inclusive-language | check-hooks-apply | check-incorrect-use-of-LoggingMixin│ -│| check-integrations-are-consistent | check-lazy-logging | check-merge-conflict |       │ -│check-newsfragments-are-valid | check-no-providers-in-core-examples |                   │ -│check-no-relative-imports | check-persist-credentials-disabled-in-github-workflows |    │ -│check-pre-commit-information-consistent | check-provide-create-sessions-imports |       │ -│check-provider-yaml-valid | check-providers-init-file-missing |                         │ -│check-providers-subpackages-init-file-exist | check-pydevd-left-in-code |               │ -│check-revision-heads-map | check-safe-filter-usage-in-html | check-setup-order |        │ -│check-start-date-not-used-in-defaults | check-system-tests-present |                    │ -│check-system-tests-tocs | check-xml | codespell | compile-www-assets |                  │ -│compile-www-assets-dev | create-missing-init-py-files-tests | debug-statements |        │ -│detect-private-key | doctoc | end-of-file-fixer | fix-encoding-pragma | flynt | identity│ -│| insert-license | isort | lint-chart-schema | lint-css | lint-dockerfile |             │ -│lint-helm-chart | lint-javascript | lint-json-schema | lint-markdown | lint-openapi |   │ -│mixed-line-ending | pretty-format-json | pydocstyle | python-no-log-warn | pyupgrade |  │ -│replace-bad-characters | rst-backticks | run-flake8 | run-mypy | run-shellcheck |       │ -│static-check-autoflake | trailing-whitespace | ts-compile-and-lint-javascript |         │ -│update-breeze-cmd-output | update-breeze-readme-config-hash | update-extras |           │ -│update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |                   │ -│update-local-yml-file | update-migration-references | update-providers-dependencies |   │ -│update-setup-cfg-file | update-spelling-wordlist-to-be-sorted |                         │ -│update-supported-versions | update-vendored-in-k8s-json-schema | update-version |       │ -│yamllint | yesqa)                                                                       │ -│--file-fList of files to run the checks on.(PATH)│ -│--all-files-aRun checks on all files.│ -│--show-diff-on-failure-sShow diff for files modified by the checks.│ -│--last-commit-cRun checks for all files in last commit. Mutually exclusive with --commit-ref.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--commit-ref-rRun checks for this commit reference only (can be any git commit-ish reference). Mutually  │ -│exclusive with --last-commit.                                                              │ -│(TEXT)                                                                                     │ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-stop.svg b/images/breeze/output-stop.svg deleted file mode 100644 index 5fc1819e95293..0000000000000 --- a/images/breeze/output-stop.svg +++ /dev/null @@ -1,111 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: stop - - - - - - - - - - -Usage: breeze stop [OPTIONS] - -Stop running breeze environment. - -╭─ Stop flags ─────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--preserve-volumes-pSkip removing volumes when stopping Breeze.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-tests.svg b/images/breeze/output-tests.svg deleted file mode 100644 index ed99d925d256e..0000000000000 --- a/images/breeze/output-tests.svg +++ /dev/null @@ -1,192 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: tests - - - - - - - - - - -Usage: breeze tests [OPTIONS] [EXTRA_PYTEST_ARGS]... - -Run the specified unit test targets. Multiple targets may be specified separated by spaces. - -╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮ -│--integrationIntegration(s) to enable when running (can be more than one).                              │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino | all)│ -│--test-typeType of test to run. Note that with Providers, you can also specify which provider tests    │ -│should be run - for example --test-type "Providers[airbyte,http]"                           │ -│(All | Always | API | Always | CLI | Core | Integration | Other | Providers | WWW | Helm |  │ -│Postgres | MySQL | Integration | Other | Quarantine)                                        │ -│--db-reset-dReset DB when entering the container.│ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ -│--limit-progress-outputLimit progress to percentage only and just show the summary when tests complete.│ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip)(TEXT)│ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =   │ -│selected).                                                                             │ -│(selected | all | skip | remove)                                                       │ -│[default: selected]                                                                    │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-verify-image.svg b/images/breeze/output-verify-image.svg deleted file mode 100644 index 44dc59c17b0fb..0000000000000 --- a/images/breeze/output-verify-image.svg +++ /dev/null @@ -1,132 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: verify-image - - - - - - - - - - -Usage: breeze verify-image [OPTIONS] [EXTRA_PYTEST_ARGS]... - -Verify CI image. - -╭─ Verify image flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--image-tag-tTag of the image when verifying it(TEXT)│ -│--pull-imagePull image is missing before attempting to verify it.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-verify-prod-image.svg b/images/breeze/output-verify-prod-image.svg deleted file mode 100644 index 242ba205e54d7..0000000000000 --- a/images/breeze/output-verify-prod-image.svg +++ /dev/null @@ -1,136 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: verify-prod-image - - - - - - - - - - -Usage: breeze verify-prod-image [OPTIONS] [EXTRA_PYTEST_ARGS]... - -Verify Production image. - -╭─ Verify image flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--image-tag-tTag of the image when verifying it(TEXT)│ -│--pull-imagePull image is missing before attempting to verify it.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--slim-imageThe image to verify is slim and non-slim tests should be skipped.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-verify-provider-packages.svg b/images/breeze/output-verify-provider-packages.svg deleted file mode 100644 index fbfd468ffafd1..0000000000000 --- a/images/breeze/output-verify-provider-packages.svg +++ /dev/null @@ -1,172 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: verify-provider-packages - - - - - - - - - - -Usage: breeze verify-provider-packages [OPTIONS] - -Verifies if all provider code is following expectations for providers. - -╭─ Provider verification flags ────────────────────────────────────────────────────────────────────────────────────────╮ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`,       │ -│`wheel`, or `sdist` if Airflow should be removed, installed from wheel packages  │ -│or sdist packages available in dist folder respectively. Implies --mount-sources│ -│`remove`.                                                                        │ -│(none | wheel | sdist | <airflow_version>)                                       │ -│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to      │ -│specify constraints for the installed version and to find newer dependencies     │ -│(TEXT)                                                                           │ -│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ -│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist' folder │ -│when entering breeze.                                                            │ -│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ -│[default: wheel]                                      │ -│--skip-constraintsDo not use constraints when installing providers.│ -│--debugDrop user in shell instead of running the command. Useful for debugging.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output-version.svg b/images/breeze/output-version.svg deleted file mode 100644 index 9d37a0a7ac852..0000000000000 --- a/images/breeze/output-version.svg +++ /dev/null @@ -1,95 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: version - - - - - - - - - - -Usage: breeze version [OPTIONS] - -Print information about version of apache-airflow-breeze. - -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/images/breeze/output_build-docs.svg b/images/breeze/output_build-docs.svg new file mode 100644 index 0000000000000..7e1a093cdbc2a --- /dev/null +++ b/images/breeze/output_build-docs.svg @@ -0,0 +1,296 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: build-docs + + + + + + + + + + +Usage: breeze build-docs [OPTIONS] + +Build documentation in the container. + +╭─ Doc flags ──────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--docs-only-dOnly build documentation.│ +│--spellcheck-only-sOnly run spell checking.│ +│--clean-buildClean inventories of Inter-Sphinx documentation and generated APIs and sphinx artifacts      │ +│before the build - useful for a clean build.                                                 │ +│--for-productionBuilds documentation for official release i.e. all links point to stable version. Implies    │ +│--clean-build│ +│--package-filterList of packages to consider.                                                                │ +│(apache-airflow | apache-airflow-providers | apache-airflow-providers-airbyte |              │ +│apache-airflow-providers-alibaba | apache-airflow-providers-amazon |                         │ +│apache-airflow-providers-apache-beam | apache-airflow-providers-apache-cassandra |           │ +│apache-airflow-providers-apache-drill | apache-airflow-providers-apache-druid |              │ +│apache-airflow-providers-apache-hdfs | apache-airflow-providers-apache-hive |                │ +│apache-airflow-providers-apache-kylin | apache-airflow-providers-apache-livy |               │ +│apache-airflow-providers-apache-pig | apache-airflow-providers-apache-pinot |                │ +│apache-airflow-providers-apache-spark | apache-airflow-providers-apache-sqoop |              │ +│apache-airflow-providers-arangodb | apache-airflow-providers-asana |                         │ +│apache-airflow-providers-celery | apache-airflow-providers-cloudant |                        │ +│apache-airflow-providers-cncf-kubernetes | apache-airflow-providers-common-sql |             │ +│apache-airflow-providers-databricks | apache-airflow-providers-datadog |                     │ +│apache-airflow-providers-dbt-cloud | apache-airflow-providers-dingding |                     │ +│apache-airflow-providers-discord | apache-airflow-providers-docker |                         │ +│apache-airflow-providers-elasticsearch | apache-airflow-providers-exasol |                   │ +│apache-airflow-providers-facebook | apache-airflow-providers-ftp |                           │ +│apache-airflow-providers-github | apache-airflow-providers-google |                          │ +│apache-airflow-providers-grpc | apache-airflow-providers-hashicorp |                         │ +│apache-airflow-providers-http | apache-airflow-providers-imap |                              │ +│apache-airflow-providers-influxdb | apache-airflow-providers-jdbc |                          │ +│apache-airflow-providers-jenkins | apache-airflow-providers-jira |                           │ +│apache-airflow-providers-microsoft-azure | apache-airflow-providers-microsoft-mssql |        │ +│apache-airflow-providers-microsoft-psrp | apache-airflow-providers-microsoft-winrm |         │ +│apache-airflow-providers-mongo | apache-airflow-providers-mysql |                            │ +│apache-airflow-providers-neo4j | apache-airflow-providers-odbc |                             │ +│apache-airflow-providers-openfaas | apache-airflow-providers-opsgenie |                      │ +│apache-airflow-providers-oracle | apache-airflow-providers-pagerduty |                       │ +│apache-airflow-providers-papermill | apache-airflow-providers-plexus |                       │ +│apache-airflow-providers-postgres | apache-airflow-providers-presto |                        │ +│apache-airflow-providers-qubole | apache-airflow-providers-redis |                           │ +│apache-airflow-providers-salesforce | apache-airflow-providers-samba |                       │ +│apache-airflow-providers-segment | apache-airflow-providers-sendgrid |                       │ +│apache-airflow-providers-sftp | apache-airflow-providers-singularity |                       │ +│apache-airflow-providers-slack | apache-airflow-providers-snowflake |                        │ +│apache-airflow-providers-sqlite | apache-airflow-providers-ssh |                             │ +│apache-airflow-providers-tableau | apache-airflow-providers-tabular |                        │ +│apache-airflow-providers-telegram | apache-airflow-providers-trino |                         │ +│apache-airflow-providers-vertica | apache-airflow-providers-yandex |                         │ +│apache-airflow-providers-zendesk | docker-stack | helm-chart)                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_ci-image.svg b/images/breeze/output_ci-image.svg new file mode 100644 index 0000000000000..9e8fab3a46361 --- /dev/null +++ b/images/breeze/output_ci-image.svg @@ -0,0 +1,111 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: ci-image + + + + + + + + + + +Usage: breeze ci-image [OPTIONS] COMMAND [ARGS]... + +Tools that developers can use to manually manage CI images + +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ CI Image tools ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│build   Build CI image. Include building multiple images for all python versions (sequentially).                  │ +│pull    Pull and optionally verify CI images - possibly in parallel for all Python versions.                      │ +│verify  Verify CI image.                                                                                          │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_ci-image_build.svg b/images/breeze/output_ci-image_build.svg new file mode 100644 index 0000000000000..18a83c2c61215 --- /dev/null +++ b/images/breeze/output_ci-image_build.svg @@ -0,0 +1,328 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: ci-image build + + + + + + + + + + +Usage: breeze ci-image build [OPTIONS] + +Build CI image. Include building multiple images for all python versions (sequentially). + +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ +│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ +│--image-tag-tTag the image after building it(TEXT)│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ +│when you build or pull image with --image-tag.                                 │ +│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ +│[default: registry]                          │ +│--force-buildForce image build no matter if it is determined as needed.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10]                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ +│--builderBuildx builder used to perform `docker buildx build` commands(TEXT)│ +│--install-providers-from-sourcesInstall providers from sources when installing.│ +│--airflow-constraints-modeMode of constraints for CI image building                              │ +│(constraints-source-providers | constraints | constraints-no-providers)│ +│[default: constraints-source-providers]                                │ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +│--python-imageIf specified this is the base python image used to build the image. Should be   │ +│something like: python:VERSION-slim-bullseye                                    │ +│(TEXT)                                                                          │ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ +│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ +│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ +│itself).                                                                        │ +│(TEXT)                                                                          │ +│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ +│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--pushPush image after building it.│ +│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│image).                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_ci-image_pull.svg b/images/breeze/output_ci-image_pull.svg new file mode 100644 index 0000000000000..3254d2d31de54 --- /dev/null +++ b/images/breeze/output_ci-image_pull.svg @@ -0,0 +1,162 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: ci-image pull + + + + + + + + + + +Usage: breeze ci-image pull [OPTIONS] [EXTRA_PYTEST_ARGS]... + +Pull and optionally verify CI images - possibly in parallel for all Python versions. + +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10]                                                   │ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--verifyVerify image.│ +│--wait-for-imageWait until image is available.│ +│*--image-tag-tTag of the image which is used to pull the image(TEXT)[required]│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when you│ +│build or pull image with --image-tag.                                                   │ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_ci-image_verify.svg b/images/breeze/output_ci-image_verify.svg new file mode 100644 index 0000000000000..bc2d7c8be9470 --- /dev/null +++ b/images/breeze/output_ci-image_verify.svg @@ -0,0 +1,132 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: ci-image verify + + + + + + + + + + +Usage: breeze ci-image verify [OPTIONS] [EXTRA_PYTEST_ARGS]... + +Verify CI image. + +╭─ Verify image flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--image-tag-tTag of the image when verifying it(TEXT)│ +│--pullPull image is missing before attempting to verify it.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_ci.svg b/images/breeze/output_ci.svg new file mode 100644 index 0000000000000..fde4c12dcad00 --- /dev/null +++ b/images/breeze/output_ci.svg @@ -0,0 +1,119 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: ci + + + + + + + + + + +Usage: breeze ci [OPTIONS] COMMAND [ARGS]... + +Tools that CI workflows use to cleanup/manage CI environment + +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ CI commands ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│fix-ownership                  Fix ownership of source files to be same as host user.                             │ +│free-space                     Free space for jobs run in CI.                                                     │ +│resource-check                 Check if available docker resources are enough.                                    │ +│selective-check                Checks what kind of tests should be run for an incoming commit.                    │ +│find-newer-dependencies        Finds which dependencies are being upgraded.                                       │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_ci_find-newer-dependencies.svg b/images/breeze/output_ci_find-newer-dependencies.svg new file mode 100644 index 0000000000000..eb6232fd4dc1b --- /dev/null +++ b/images/breeze/output_ci_find-newer-dependencies.svg @@ -0,0 +1,140 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: ci find-newer-dependencies + + + + + + + + + + +Usage: breeze ci find-newer-dependencies [OPTIONS] + +Finds which dependencies are being upgraded. + +╭─ Find newer dependencies flags ──────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--timezoneTimezone to use during the check(TEXT)│ +│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ +│specify constraints for the installed version and to find newer dependencies   │ +│(TEXT)                                                                         │ +│--updated-on-or-afterDate when the release was updated after(TEXT)│ +│--max-ageMax age of the last release (used if no updated-on-or-after if specified)│ +│(INTEGER)                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_ci_fix-ownership.svg b/images/breeze/output_ci_fix-ownership.svg new file mode 100644 index 0000000000000..c8b9266bd1ea9 --- /dev/null +++ b/images/breeze/output_ci_fix-ownership.svg @@ -0,0 +1,116 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: ci fix-ownership + + + + + + + + + + +Usage: breeze ci fix-ownership [OPTIONS] + +Fix ownership of source files to be same as host user. + +╭─ Fix ownership flags ────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--use-sudoUse sudo instead of docker image to fix the ownership. You need to be a `sudoer` to run it│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_ci_free-space.svg b/images/breeze/output_ci_free-space.svg new file mode 100644 index 0000000000000..d6337eaac8f8d --- /dev/null +++ b/images/breeze/output_ci_free-space.svg @@ -0,0 +1,104 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: ci free-space + + + + + + + + + + +Usage: breeze ci free-space [OPTIONS] + +Free space for jobs run in CI. + +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_ci_resource-check.svg b/images/breeze/output_ci_resource-check.svg new file mode 100644 index 0000000000000..2f4c13a6bbc2e --- /dev/null +++ b/images/breeze/output_ci_resource-check.svg @@ -0,0 +1,99 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: ci resource-check + + + + + + + + + + +Usage: breeze ci resource-check [OPTIONS] + +Check if available docker resources are enough. + +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_ci_selective-check.svg b/images/breeze/output_ci_selective-check.svg new file mode 100644 index 0000000000000..2f117e63d068f --- /dev/null +++ b/images/breeze/output_ci_selective-check.svg @@ -0,0 +1,144 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: ci selective-check + + + + + + + + + + +Usage: breeze ci selective-check [OPTIONS] + +Checks what kind of tests should be run for an incoming commit. + +╭─ Selective check flags ──────────────────────────────────────────────────────────────────────────────────────────────╮ +│--commit-refCommit-ish reference to the commit that should be checked(TEXT)│ +│--pr-labelsPython array formatted PR labels assigned to the PR(TEXT)│ +│--default-branchBranch against which the PR should be run(TEXT)[default: main]│ +│--default-constraints-branchConstraints Branch against which the PR should be run(TEXT)│ +│[default: constraints-main]                          │ +│--github-event-nameName of the GitHub event that triggered the check                                   │ +│(pull_request | pull_request_review | pull_request_target | pull_request_workflow | │ +│push | schedule | workflow_dispatch | workflow_run)                                 │ +│[default: pull_request]                                                             │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_cleanup.svg b/images/breeze/output_cleanup.svg new file mode 100644 index 0000000000000..50acd7348c52b --- /dev/null +++ b/images/breeze/output_cleanup.svg @@ -0,0 +1,120 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: cleanup + + + + + + + + + + +Usage: breeze cleanup [OPTIONS] + +Cleans the cache of parameters, docker cache and optionally built CI/PROD images. + +╭─ Cleanup flags ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--allAlso remove currently downloaded Breeze images.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_compile-www-assets.svg b/images/breeze/output_compile-www-assets.svg new file mode 100644 index 0000000000000..26ba96de9c537 --- /dev/null +++ b/images/breeze/output_compile-www-assets.svg @@ -0,0 +1,115 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: compile-www-assets + + + + + + + + + + +Usage: breeze compile-www-assets [OPTIONS] + +Compiles www assets. + +╭─ Compile www assets flag ────────────────────────────────────────────────────────────────────────────────────────────╮ +│--devRun development version of assets compilation - it will not quit and automatically recompile assets        │ +│on-the-fly when they are changed.                                                                          │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_exec.svg b/images/breeze/output_exec.svg new file mode 100644 index 0000000000000..2541c71f65db0 --- /dev/null +++ b/images/breeze/output_exec.svg @@ -0,0 +1,99 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: exec + + + + + + + + + + +Usage: breeze exec [OPTIONS] [EXEC_ARGS]... + +Joins the interactive shell of running airflow container. + +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_prod-image.svg b/images/breeze/output_prod-image.svg new file mode 100644 index 0000000000000..a47af3fe6b94f --- /dev/null +++ b/images/breeze/output_prod-image.svg @@ -0,0 +1,119 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: prod-image + + + + + + + + + + +Usage: breeze prod-image [OPTIONS] COMMAND [ARGS]... + +Tools that developers can use to manually manage PROD images + +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Production Image tools ─────────────────────────────────────────────────────────────────────────────────────────────╮ +│build  Build Production image. Include building multiple images for all or selected Python versions sequentially. │ +│verify Verify Production image.                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│pull Pull and optionally verify Production images - possibly in parallel for all Python versions.                 │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_prod-image_build.svg b/images/breeze/output_prod-image_build.svg new file mode 100644 index 0000000000000..92e71397a5a9a --- /dev/null +++ b/images/breeze/output_prod-image_build.svg @@ -0,0 +1,392 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: prod-image build + + + + + + + + + + +Usage: breeze prod-image build [OPTIONS] + +Build Production image. Include building multiple images for all or selected Python versions sequentially. + +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ +│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ +│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ +│--image-tag-tTag the image after building it(TEXT)│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ +│when you build or pull image with --image-tag.                                 │ +│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ +│[default: registry]                          │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10]                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ +│--builderBuildx builder used to perform `docker buildx build` commands(TEXT)│ +│--install-providers-from-sourcesInstall providers from sources when installing.│ +│--airflow-extrasExtras to install by default.                                                   │ +│(TEXT)                                                                          │ +│[default:                                                                       │ +│amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google…│ +│--airflow-constraints-modeMode of constraints for PROD image building                            │ +│(constraints | constraints-no-providers | constraints-source-providers)│ +│[default: constraints]                                                 │ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +│--python-imageIf specified this is the base python image used to build the image. Should be   │ +│something like: python:VERSION-slim-bullseye                                    │ +│(TEXT)                                                                          │ +│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ +│itself).                                                                        │ +│(TEXT)                                                                          │ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ +│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ +│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ +│--install-packages-from-contextInstall wheels from local docker-context-files when building image.       │ +│Implies --disable-airflow-repo-cache.                                     │ +│--cleanup-contextClean up docker context files before running build (cannot be used        │ +│together with --install-packages-from-context).                           │ +│--disable-mysql-client-installationDo not install MySQL client.│ +│--disable-mssql-client-installationDo not install MsSQl client.│ +│--disable-postgres-client-installationDo not install Postgres client.│ +│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ +│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ +│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--pushPush image after building it.│ +│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│image).                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_prod-image_pull.svg b/images/breeze/output_prod-image_pull.svg new file mode 100644 index 0000000000000..8728c77cba8e8 --- /dev/null +++ b/images/breeze/output_prod-image_pull.svg @@ -0,0 +1,162 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: prod-image pull + + + + + + + + + + +Usage: breeze prod-image pull [OPTIONS] [EXTRA_PYTEST_ARGS]... + +Pull and optionally verify Production images - possibly in parallel for all Python versions. + +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10]                                                   │ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│*--image-tag-tTag of the image which is used to pull the image(TEXT)[required]│ +│--wait-for-imageWait until image is available.│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when you│ +│build or pull image with --image-tag.                                                   │ +│--verifyVerify image.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_prod-image_verify.svg b/images/breeze/output_prod-image_verify.svg new file mode 100644 index 0000000000000..9fad7ce5745d2 --- /dev/null +++ b/images/breeze/output_prod-image_verify.svg @@ -0,0 +1,136 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: prod-image verify + + + + + + + + + + +Usage: breeze prod-image verify [OPTIONS] [EXTRA_PYTEST_ARGS]... + +Verify Production image. + +╭─ Verify image flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--slim-imageThe image to verify is slim and non-slim tests should be skipped.│ +│--image-tag-tTag of the image when verifying it(TEXT)│ +│--pullPull image is missing before attempting to verify it.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_release-management.svg b/images/breeze/output_release-management.svg new file mode 100644 index 0000000000000..66af753517d41 --- /dev/null +++ b/images/breeze/output_release-management.svg @@ -0,0 +1,123 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: release-management + + + + + + + + + + +Usage: breeze release-management [OPTIONS] COMMAND [ARGS]... + +Tools that release managers can use to prepare and manage Airflow releases + +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│generate-constraints             Generates pinned constraint files with all extras from setup.py in parallel.     │ +│prepare-airflow-package          Prepare sdist/whl package of Airflow.                                            │ +│prepare-provider-documentation   Prepare CHANGELOG, README and COMMITS information for providers.                 │ +│prepare-provider-packages        Prepare sdist/whl packages of Airflow Providers.                                 │ +│release-prod-images              Release production images to DockerHub (needs DockerHub permissions).            │ +│verify-provider-packages         Verifies if all provider code is following expectations for providers.           │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_release-management_generate-constraints.svg b/images/breeze/output_release-management_generate-constraints.svg new file mode 100644 index 0000000000000..763aa3d1a54e0 --- /dev/null +++ b/images/breeze/output_release-management_generate-constraints.svg @@ -0,0 +1,180 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: release-management generate-constraints + + + + + + + + + + +Usage: breeze release-management generate-constraints [OPTIONS] + +Generates pinned constraint files with all extras from setup.py in parallel. + +╭─ Generate constraints flags ─────────────────────────────────────────────────────────────────────────────────────────╮ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip)│ +│(TEXT)                                                                        │ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--airflow-constraints-modeMode of constraints for CI image building                              │ +│(constraints-source-providers | constraints | constraints-no-providers)│ +│[default: constraints-source-providers]                                │ +│--debugDrop user in shell instead of running the command. Useful for debugging.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10]                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_release-management_prepare-airflow-package.svg b/images/breeze/output_release-management_prepare-airflow-package.svg new file mode 100644 index 0000000000000..ad957a4ebd070 --- /dev/null +++ b/images/breeze/output_release-management_prepare-airflow-package.svg @@ -0,0 +1,124 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: release-management prepare-airflow-package + + + + + + + + + + +Usage: breeze release-management prepare-airflow-package [OPTIONS] + +Prepare sdist/whl package of Airflow. + +╭─ Package flags ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--package-formatFormat of packages.(wheel | sdist | both)[default: wheel]│ +│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--debugDrop user in shell instead of running the command. Useful for debugging.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_release-management_prepare-provider-documentation.svg b/images/breeze/output_release-management_prepare-provider-documentation.svg new file mode 100644 index 0000000000000..f7f75a21dcc6b --- /dev/null +++ b/images/breeze/output_release-management_prepare-provider-documentation.svg @@ -0,0 +1,184 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: release-management prepare-provider-documentation + + + + + + + + + + +Usage: breeze release-management prepare-provider-documentation [OPTIONS] [airbyte | alibaba | amazon | apache.beam | +                                                                apache.cassandra | apache.drill | apache.druid | +                                                                apache.hdfs | apache.hive | apache.kylin | apache.livy +                                                                | apache.pig | apache.pinot | apache.spark | +                                                                apache.sqoop | arangodb | asana | celery | cloudant | +                                                                cncf.kubernetes | common.sql | databricks | datadog | +                                                                dbt.cloud | dingding | discord | docker | +                                                                elasticsearch | exasol | facebook | ftp | github | +                                                                google | grpc | hashicorp | http | imap | influxdb | +                                                                jdbc | jenkins | jira | microsoft.azure | +                                                                microsoft.mssql | microsoft.psrp | microsoft.winrm | +                                                                mongo | mysql | neo4j | odbc | openfaas | opsgenie | +                                                                oracle | pagerduty | papermill | plexus | postgres | +                                                                presto | qubole | redis | salesforce | samba | segment +                                                                | sendgrid | sftp | singularity | slack | snowflake | +                                                                sqlite | ssh | tableau | tabular | telegram | trino | +                                                                vertica | yandex | zendesk]... + +Prepare CHANGELOG, README and COMMITS information for providers. + +╭─ Provider documentation preparation flags ───────────────────────────────────────────────────────────────────────────╮ +│--debugDrop user in shell instead of running the command. Useful for debugging.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_release-management_prepare-provider-packages.svg b/images/breeze/output_release-management_prepare-provider-packages.svg new file mode 100644 index 0000000000000..cca87e66887f0 --- /dev/null +++ b/images/breeze/output_release-management_prepare-provider-packages.svg @@ -0,0 +1,184 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: release-management prepare-provider-packages + + + + + + + + + + +Usage: breeze release-management prepare-provider-packages [OPTIONS] [airbyte | alibaba | amazon | apache.beam | +                                                           apache.cassandra | apache.drill | apache.druid | +                                                           apache.hdfs | apache.hive | apache.kylin | apache.livy | +                                                           apache.pig | apache.pinot | apache.spark | apache.sqoop | +                                                           arangodb | asana | celery | cloudant | cncf.kubernetes | +                                                           common.sql | databricks | datadog | dbt.cloud | dingding | +                                                           discord | docker | elasticsearch | exasol | facebook | ftp +                                                           | github | google | grpc | hashicorp | http | imap | +                                                           influxdb | jdbc | jenkins | jira | microsoft.azure | +                                                           microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo +                                                           | mysql | neo4j | odbc | openfaas | opsgenie | oracle | +                                                           pagerduty | papermill | plexus | postgres | presto | qubole +                                                           | redis | salesforce | samba | segment | sendgrid | sftp | +                                                           singularity | slack | snowflake | sqlite | ssh | tableau | +                                                           tabular | telegram | trino | vertica | yandex | zendesk]... + +Prepare sdist/whl packages of Airflow Providers. + +╭─ Package flags ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--package-formatFormat of packages.(wheel | sdist | both)[default: wheel]│ +│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ +│--package-list-fileRead list of packages from text file (one package per line)(FILENAME)│ +│--debugDrop user in shell instead of running the command. Useful for debugging.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_release-management_release-prod-images.svg b/images/breeze/output_release-management_release-prod-images.svg new file mode 100644 index 0000000000000..ef945e9fd0ed2 --- /dev/null +++ b/images/breeze/output_release-management_release-prod-images.svg @@ -0,0 +1,158 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: release-management release-prod-images + + + + + + + + + + +Usage: breeze release-management release-prod-images [OPTIONS] + +Release production images to DockerHub (needs DockerHub permissions). + +╭─ Release PROD IMAGE flags ───────────────────────────────────────────────────────────────────────────────────────────╮ +│*--airflow-versionAirflow version to release (2.3.0, 2.3.0rc1 etc.)(TEXT)[required]│ +│--dockerhub-repoDockerHub repository for the images(TEXT)[default: apache/airflow]│ +│--slim-imagesWhether to prepare slim images instead of the regular ones.│ +│--limit-pythonSpecific python to build slim images for (if not specified - the images are built for all   │ +│available python versions)                                                                  │ +│(3.7 | 3.8 | 3.9 | 3.10)                                                                    │ +│--limit-platformSpecific platform to build images for (if not specified, multiplatform images will be built.│ +│(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)                                       │ +│[default: linux/amd64,linux/arm64]                                                          │ +│--skip-latestWhether to skip publishing the latest images (so that 'latest' images are not updated). This│ +│should only be used if you release image for previous branches. Automatically set when      │ +│rc/alpha/beta images are built.                                                             │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_release-management_verify-provider-packages.svg b/images/breeze/output_release-management_verify-provider-packages.svg new file mode 100644 index 0000000000000..02b65c4c3c232 --- /dev/null +++ b/images/breeze/output_release-management_verify-provider-packages.svg @@ -0,0 +1,172 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: release-management verify-provider-packages + + + + + + + + + + +Usage: breeze release-management verify-provider-packages [OPTIONS] + +Verifies if all provider code is following expectations for providers. + +╭─ Provider verification flags ────────────────────────────────────────────────────────────────────────────────────────╮ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`,       │ +│`wheel`, or `sdist` if Airflow should be removed, installed from wheel packages  │ +│or sdist packages available in dist folder respectively. Implies --mount-sources│ +│`remove`.                                                                        │ +│(none | wheel | sdist | <airflow_version>)                                       │ +│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to      │ +│specify constraints for the installed version and to find newer dependencies     │ +│(TEXT)                                                                           │ +│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ +│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist' folder │ +│when entering breeze.                                                            │ +│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ +│[default: wheel]                                      │ +│--skip-constraintsDo not use constraints when installing providers.│ +│--debugDrop user in shell instead of running the command. Useful for debugging.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_setup.svg b/images/breeze/output_setup.svg new file mode 100644 index 0000000000000..29a138ecb40aa --- /dev/null +++ b/images/breeze/output_setup.svg @@ -0,0 +1,127 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: setup + + + + + + + + + + +Usage: breeze setup [OPTIONS] COMMAND [ARGS]... + +Tools that developers can use to configure Breeze + +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│autocomplete             Enables autocompletion of breeze commands.                                               │ +│command-hash-export      Outputs hash of all click commands to file or stdout if `-` is used (useful to see if    │ +│images should be regenerated).                                                           │ +│config                   Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).                       │ +│regenerate-command-imagesRegenerate breeze command images.                                                        │ +│self-upgrade             Self upgrade Breeze.                                                                     │ +│version                  Print information about version of apache-airflow-breeze.                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_setup_autocomplete.svg b/images/breeze/output_setup_autocomplete.svg new file mode 100644 index 0000000000000..9edf5178d658f --- /dev/null +++ b/images/breeze/output_setup_autocomplete.svg @@ -0,0 +1,116 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: setup autocomplete + + + + + + + + + + +Usage: breeze setup autocomplete [OPTIONS] + +Enables autocompletion of breeze commands. + +╭─ Setup autocomplete flags ───────────────────────────────────────────────────────────────────────────────────────────╮ +│--force-fForce autocomplete setup even if already setup before (overrides the setup).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_setup_command-hash-export.svg b/images/breeze/output_setup_command-hash-export.svg new file mode 100644 index 0000000000000..3f9412fce69da --- /dev/null +++ b/images/breeze/output_setup_command-hash-export.svg @@ -0,0 +1,95 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: setup command-hash-export + + + + + + + + + + +Usage: breeze setup command-hash-export [OPTIONS] OUTPUT + +Outputs hash of all click commands to file or stdout if `-` is used (useful to see if images should be regenerated). + +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_setup_config.svg b/images/breeze/output_setup_config.svg new file mode 100644 index 0000000000000..4d6c2dbba5af0 --- /dev/null +++ b/images/breeze/output_setup_config.svg @@ -0,0 +1,144 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: setup config + + + + + + + + + + +Usage: breeze setup config [OPTIONS] + +Show/update configuration (Python, Backend, Cheatsheet, ASCIIART). + +╭─ Config flags ───────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)│ +│[default: sqlite]       │ +│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--cheatsheet/--no-cheatsheet-C/-cEnable/disable cheatsheet.│ +│--asciiart/--no-asciiart-A/-aEnable/disable ASCIIart.│ +│--colour/--no-colourEnable/disable Colour mode (useful for colour blind-friendly communication).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_setup_regenerate-command-images.svg b/images/breeze/output_setup_regenerate-command-images.svg new file mode 100644 index 0000000000000..f2ee84496c46a --- /dev/null +++ b/images/breeze/output_setup_regenerate-command-images.svg @@ -0,0 +1,99 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: setup regenerate-command-images + + + + + + + + + + +Usage: breeze setup regenerate-command-images [OPTIONS] + +Regenerate breeze command images. + +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_setup_self-upgrade.svg b/images/breeze/output_setup_self-upgrade.svg new file mode 100644 index 0000000000000..810a3fc41c956 --- /dev/null +++ b/images/breeze/output_setup_self-upgrade.svg @@ -0,0 +1,107 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: setup self-upgrade + + + + + + + + + + +Usage: breeze setup self-upgrade [OPTIONS] + +Self upgrade Breeze. + +╭─ Self-upgrade flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--use-current-airflow-sources-aUse current workdir Airflow sources for upgrade.│ +│--force-fForce upgrade without asking question to the user.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_setup_version.svg b/images/breeze/output_setup_version.svg new file mode 100644 index 0000000000000..c8743e18abe7b --- /dev/null +++ b/images/breeze/output_setup_version.svg @@ -0,0 +1,95 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: setup version + + + + + + + + + + +Usage: breeze setup version [OPTIONS] + +Print information about version of apache-airflow-breeze. + +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_shell.svg b/images/breeze/output_shell.svg new file mode 100644 index 0000000000000..966062f8418fd --- /dev/null +++ b/images/breeze/output_shell.svg @@ -0,0 +1,256 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: shell + + + + + + + + + + +Usage: breeze shell [OPTIONS] [EXTRA_ARGS]... + +Enter breeze environment. this is the default command use when no other is selected. + +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                            │ +│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ +│all)                                                                                     │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`,     │ +│`wheel`, or `sdist` if Airflow should be removed, installed from wheel packages│ +│or sdist packages available in dist folder respectively. Implies               │ +│--mount-sources `remove`.                                                      │ +│(none | wheel | sdist | <airflow_version>)                                     │ +│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ +│specify constraints for the installed version and to find newer dependencies   │ +│(TEXT)                                                                         │ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ +│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ +│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist'      │ +│folder when entering breeze.                                                   │ +│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ +│[default: wheel]                                      │ +│--force-buildForce image build no matter if it is determined as needed.│ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip)│ +│(TEXT)                                                                        │ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed      │ +│(default = selected).                                                          │ +│(selected | all | skip | remove)                                               │ +│[default: selected]                                                            │ +│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_start-airflow.svg b/images/breeze/output_start-airflow.svg new file mode 100644 index 0000000000000..e3d57d29e15c9 --- /dev/null +++ b/images/breeze/output_start-airflow.svg @@ -0,0 +1,264 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: start-airflow + + + + + + + + + + +Usage: breeze start-airflow [OPTIONS] [EXTRA_ARGS]... + +Enter breeze environment and starts all Airflow components in the tmux session. + +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--load-example-dags-eEnable configuration to load example DAGs when starting Airflow.│ +│--load-default-connections-cEnable configuration to load default connections when starting Airflow.│ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ +│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                       │ +│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino│ +│| all)                                                                              │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`,     │ +│`wheel`, or `sdist` if Airflow should be removed, installed from wheel packages│ +│or sdist packages available in dist folder respectively. Implies               │ +│--mount-sources `remove`.                                                      │ +│(none | wheel | sdist | <airflow_version>)                                     │ +│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ +│specify constraints for the installed version and to find newer dependencies   │ +│(TEXT)                                                                         │ +│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ +│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist'      │ +│folder when entering breeze.                                                   │ +│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ +│[default: wheel]                                      │ +│--force-buildForce image build no matter if it is determined as needed.│ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip)│ +│(TEXT)                                                                        │ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed      │ +│(default = selected).                                                          │ +│(selected | all | skip | remove)                                               │ +│[default: selected]                                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_static-checks.svg b/images/breeze/output_static-checks.svg new file mode 100644 index 0000000000000..b9f4790fd2e5c --- /dev/null +++ b/images/breeze/output_static-checks.svg @@ -0,0 +1,264 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: static-checks + + + + + + + + + + +Usage: breeze static-checks [OPTIONS] [PRECOMMIT_ARGS]... + +Run static checks. + +╭─ Pre-commit flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--type-tType(s) of the static checks to run (multiple can be added).                            │ +│(all | black | blacken-docs | check-airflow-2-2-compatibility |                         │ +│check-airflow-config-yaml-consistent | check-apache-license-rat |                       │ +│check-base-operator-partial-arguments | check-base-operator-usage |                     │ +│check-boring-cyborg-configuration | check-breeze-top-dependencies-limited |             │ +│check-builtin-literals | check-changelog-has-no-duplicates |                            │ +│check-daysago-import-from-utils | check-docstring-param-types | check-example-dags-urls │ +│| check-executables-have-shebangs | check-extra-packages-references | check-extras-order│ +│| check-for-inclusive-language | check-hooks-apply | check-incorrect-use-of-LoggingMixin│ +│| check-integrations-are-consistent | check-lazy-logging | check-merge-conflict |       │ +│check-newsfragments-are-valid | check-no-providers-in-core-examples |                   │ +│check-no-relative-imports | check-persist-credentials-disabled-in-github-workflows |    │ +│check-pre-commit-information-consistent | check-provide-create-sessions-imports |       │ +│check-provider-yaml-valid | check-providers-init-file-missing |                         │ +│check-providers-subpackages-init-file-exist | check-pydevd-left-in-code |               │ +│check-revision-heads-map | check-safe-filter-usage-in-html | check-setup-order |        │ +│check-start-date-not-used-in-defaults | check-system-tests-present |                    │ +│check-system-tests-tocs | check-xml | codespell | compile-www-assets |                  │ +│compile-www-assets-dev | create-missing-init-py-files-tests | debug-statements |        │ +│detect-private-key | doctoc | end-of-file-fixer | fix-encoding-pragma | flynt | identity│ +│| insert-license | isort | lint-chart-schema | lint-css | lint-dockerfile |             │ +│lint-helm-chart | lint-javascript | lint-json-schema | lint-markdown | lint-openapi |   │ +│mixed-line-ending | pretty-format-json | pydocstyle | python-no-log-warn | pyupgrade |  │ +│replace-bad-characters | rst-backticks | run-flake8 | run-mypy | run-shellcheck |       │ +│static-check-autoflake | trailing-whitespace | ts-compile-and-lint-javascript |         │ +│update-breeze-cmd-output | update-breeze-readme-config-hash | update-extras |           │ +│update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |                   │ +│update-local-yml-file | update-migration-references | update-providers-dependencies |   │ +│update-setup-cfg-file | update-spelling-wordlist-to-be-sorted |                         │ +│update-supported-versions | update-vendored-in-k8s-json-schema | update-version |       │ +│yamllint | yesqa)                                                                       │ +│--file-fList of files to run the checks on.(PATH)│ +│--all-files-aRun checks on all files.│ +│--show-diff-on-failure-sShow diff for files modified by the checks.│ +│--last-commit-cRun checks for all files in last commit. Mutually exclusive with --commit-ref.│ +│--commit-ref-rRun checks for this commit reference only (can be any git commit-ish reference).        │ +│Mutually exclusive with --last-commit.                                                  │ +│(TEXT)                                                                                  │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_stop.svg b/images/breeze/output_stop.svg new file mode 100644 index 0000000000000..54785bed03598 --- /dev/null +++ b/images/breeze/output_stop.svg @@ -0,0 +1,111 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: stop + + + + + + + + + + +Usage: breeze stop [OPTIONS] + +Stop running breeze environment. + +╭─ Stop flags ─────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--preserve-volumes-pSkip removing volumes when stopping Breeze.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_testing.svg b/images/breeze/output_testing.svg new file mode 100644 index 0000000000000..5ed3d128061ff --- /dev/null +++ b/images/breeze/output_testing.svg @@ -0,0 +1,107 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: testing + + + + + + + + + + +Usage: breeze testing [OPTIONS] COMMAND [ARGS]... + +Tools that developers can use to run tests + +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│tests                Run the specified unit test targets. Multiple targets may be specified separated by spaces.  │ +│docker-compose-tests Run docker-compose tests.                                                                    │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_testing_docker-compose-tests.svg b/images/breeze/output_testing_docker-compose-tests.svg new file mode 100644 index 0000000000000..1337fec56d1fb --- /dev/null +++ b/images/breeze/output_testing_docker-compose-tests.svg @@ -0,0 +1,128 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: testing docker-compose-tests + + + + + + + + + + +Usage: breeze testing docker-compose-tests [OPTIONS] [EXTRA_PYTEST_ARGS]... + +Run docker-compose tests. + +╭─ Docker-compose tests flag ──────────────────────────────────────────────────────────────────────────────────────────╮ +│--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip)(TEXT)│ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_testing_tests.svg b/images/breeze/output_testing_tests.svg new file mode 100644 index 0000000000000..207aab3dd6835 --- /dev/null +++ b/images/breeze/output_testing_tests.svg @@ -0,0 +1,192 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: testing tests + + + + + + + + + + +Usage: breeze testing tests [OPTIONS] [EXTRA_PYTEST_ARGS]... + +Run the specified unit test targets. Multiple targets may be specified separated by spaces. + +╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮ +│--integrationIntegration(s) to enable when running (can be more than one).                              │ +│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino | all)│ +│--test-typeType of test to run. Note that with Providers, you can also specify which provider tests    │ +│should be run - for example --test-type "Providers[airbyte,http]"                           │ +│(All | Always | API | Always | CLI | Core | Integration | Other | Providers | WWW | Helm |  │ +│Postgres | MySQL | Integration | Other | Quarantine)                                        │ +│--db-reset-dReset DB when entering the container.│ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ +│--limit-progress-outputLimit progress to percentage only and just show the summary when tests complete.│ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip)(TEXT)│ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =   │ +│selected).                                                                             │ +│(selected | all | skip | remove)                                                       │ +│[default: selected]                                                                    │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py b/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py index 360672dc8f2b3..57a43e85f8030 100755 --- a/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py +++ b/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py @@ -20,6 +20,7 @@ import sys from pathlib import Path from subprocess import check_call, check_output, run +from typing import List from rich.console import Console @@ -32,12 +33,12 @@ PREAMBLE = """ # This file is automatically generated by pre-commit. If you have a conflict with this file -# Please do not solve it but run `breeze regenerate-command-images`. +# Please do not solve it but run `breeze setup regenerate-command-images`. # This command should fix the conflict and regenerate help images that you have conflict with. """ -def get_command_list(): +def get_command_list() -> List[str]: comp_env = os.environ.copy() comp_env['COMP_WORDS'] = "" comp_env['COMP_CWORD'] = "0" @@ -46,6 +47,17 @@ def get_command_list(): return [x.split(",")[1] for x in result.splitlines(keepends=False)] +def get_subcommand_list(subcommand: str) -> List[str]: + comp_env = os.environ.copy() + comp_env['COMP_WORDS'] = f"breeze {subcommand}" + comp_env['COMP_CWORD'] = "2" + comp_env['_BREEZE_COMPLETE'] = 'bash_complete' + result = check_output('breeze', env=comp_env, text=True) + if result.strip() == '': + return [] + return [x.split(",")[1] for x in result.splitlines(keepends=False)] + + def print_help_for_all_commands(): console = Console(width=int(SCREENSHOT_WIDTH), color_system="standard") env = os.environ.copy() @@ -59,6 +71,7 @@ def print_help_for_all_commands(): [ sys.executable, str(BREEZE_SOURCES_DIR / "airflow_breeze" / "breeze.py"), + "setup", "command-hash-export", "-", ], @@ -91,9 +104,18 @@ def print_help_for_all_commands(): env['AIRFLOW_SOURCES_ROOT'] = str(AIRFLOW_SOURCES_DIR) env['RECORD_BREEZE_WIDTH'] = SCREENSHOT_WIDTH env['RECORD_BREEZE_TITLE'] = f"Command: {command}" - env['RECORD_BREEZE_OUTPUT_FILE'] = str(BREEZE_IMAGES_DIR / f"output-{command}.svg") + env['RECORD_BREEZE_OUTPUT_FILE'] = str(BREEZE_IMAGES_DIR / f"output_{command}.svg") env['TERM'] = "xterm-256color" check_call(["breeze", command, "--help"], env=env) + for subcommand in get_subcommand_list(command): + env = os.environ.copy() + env['AIRFLOW_SOURCES_ROOT'] = str(AIRFLOW_SOURCES_DIR) + env['RECORD_BREEZE_WIDTH'] = SCREENSHOT_WIDTH + env['RECORD_BREEZE_TITLE'] = f"Command: {command} {subcommand}" + env['RECORD_BREEZE_OUTPUT_FILE'] = str(BREEZE_IMAGES_DIR / f"output_{command}_{subcommand}.svg") + env['TERM'] = "xterm-256color" + check_call(["breeze", command, subcommand, "--help"], env=env) + hash_file_path.write_text(new_hash_dump) @@ -101,12 +123,12 @@ def verify_all_commands_described_in_docs(): console = Console(width=int(SCREENSHOT_WIDTH), color_system="standard") errors = [] doc_content = (AIRFLOW_SOURCES_DIR / "BREEZE.rst").read_text() - for file in os.listdir(BREEZE_IMAGES_DIR): - if file.startswith("output-") and file.endswith(".svg"): - command = file[len("output-") : -len(".svg")] + for file_name in os.listdir(BREEZE_IMAGES_DIR): + if file_name.startswith("output_") and file_name.endswith(".svg"): + command = file_name[len("output_") : -len(".svg")] if command == "breeze-commands": continue - if file not in doc_content: + if file_name not in doc_content: errors.append(command) else: console.print(f"[green]OK. The {command} screenshot is embedded in BREEZE.rst.") @@ -116,7 +138,8 @@ def verify_all_commands_described_in_docs(): console.print(f" * [red]{command}[/]") console.print() console.print( - "[bright_yellow]Make sure you describe it and embed ./images/breeze/output-.svg " + "[bright_yellow]Make sure you describe it and embed " + "./images/breeze/output_[_].svg " "screenshot as image in the BREEZE.rst file.[/]" ) sys.exit(1) @@ -128,6 +151,6 @@ def verify_all_commands_described_in_docs(): console = Console(width=int(SCREENSHOT_WIDTH), color_system="standard") if os.environ.get('CI'): console.print( - "\n\n[yellow] If you see that images got re-generatead in CI, please run:[/]" - "\n\n breeze regenerate-command-images\n\n\n" + "\n\n[yellow] If you see that images got re-generated in CI, please run:[/]" + "\n\n breeze setup regenerate-command-images\n\n\n" )