Convert Helm tests to use the new Python Breeeze (apache#25678)

This PR converts the Helm tests to use the new Python Breeze.
It has all the features of previous Kind breeze command and more.

All the commands are now grupped under k8s group and they are very
easy to use locally (even easier than the previous version).

The CI part is also converted and simplified - i.e. the upgrade
test is now much faster (only tests one upgrade per job and it
runs withing the original Helm/Kubernetes tests jobs so it will
not have the cluster creation overhead.

Most importantly - this is almost the last step before we can get
rid of the old legacy breeze code and one that we can get rid of
the `./breeze-legacy` script because all functionality from the
old breeze has been moved to the Python version with this change.

This removal allows us also to remove a lot of the common library
bash code that is not used any more anywhere - even in CI.

The only change left is running regular tests in parallel.

Closes: apache#23085

.gitattributes

@@ -1,6 +1,3 @@
-breeze export-ignore
-breeze-legacy export-ignore
-breeze-complete export-ignore
 clients export-ignore
 clients export-ignore
 dev export-ignore

.github/CODEOWNERS

@@ -70,8 +70,6 @@
 
 # Dev tools
 /.github/workflows/ @potiuk @ashb @kaxil
-breeze @potiuk
-breeze-complete @potiuk
 Dockerfile @potiuk @ashb @mik-laj
 Dockerfile.ci @potiuk @ashb
 /dev/ @potiuk @ashb @jedcunningham

.github/workflows/ci.yml

@@ -133,13 +133,10 @@ jobs:
       all-python-versions-list-as-string: >-
         ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
       default-python-version: ${{ steps.selective-checks.outputs.default-python-version }}
-      min-max-python-versions-as-string: >-
-        ${{ steps.selective-checks.outputs.min-max-python-versions-as-string }}
       kubernetes-versions-list-as-string: >-
         ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }}
+      kubernetes-combos: ${{ steps.selective-checks.outputs.kubernetes-combos }}
       default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }}
-      min-max-kubernetes-versions-as-string: >-
-        ${{ steps.selective-checks.outputs.min-max-kubernetes-versions-as-string }}
       postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }}
       default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }}
       mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }}
@@ -1441,7 +1438,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
 
   tests-kubernetes:
     timeout-minutes: 240
-    name: Helm Chart; ${{matrix.executor}} - ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
+    name: "Helm: ${{matrix.executor}} - ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}"
     runs-on: ${{ needs.build-info.outputs.runs-on }}
     needs: [build-info, wait-for-prod-images]
     strategy:
@@ -1450,17 +1447,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       fail-fast: false
     env:
       RUNS_ON: ${{ needs.build-info.outputs.runs-on }}
-      BACKEND: postgres
-      RUN_TESTS: "true"
-      RUNTIME: "kubernetes"
-      KUBERNETES_MODE: "image"
-      EXECUTOR: ${{matrix.executor}}
-      KIND_VERSION: "${{ needs.build-info.outputs.default-kind-version }}"
-      HELM_VERSION: "${{ needs.build-info.outputs.default-helm-version }}"
-      CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
-        ${{needs.build-info.outputs.python-versions-list-as-string}}
-      CURRENT_KUBERNETES_VERSIONS_AS_STRING: >
-        ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
     if: >
       ( needs.build-info.outputs.run-kubernetes-tests == 'true' ||
       needs.build-info.outputs.needs-helm-tests == 'true' ) &&
@@ -1489,98 +1475,63 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       - name: "Cache bin folder with tools for kubernetes testing"
         uses: actions/cache@v3
         with:
-          path: ".build/kubernetes-bin"
-          key: "kubernetes-binaries
--${{ needs.build-info.outputs.default-kind-version }}\
--${{ needs.build-info.outputs.default-helm-version }}"
-          restore-keys: "kubernetes-binaries"
-      - name: "Kubernetes Tests"
-        run: ./scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh
-        env:
-          PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}"
-      - name: "Upload KinD logs"
-        uses: actions/upload-artifact@v3
-        if: failure() || cancelled()
-        with:
-          name: kind-logs-${{matrix.executor}}
-          path: /tmp/kind_logs_*
-          retention-days: 7
-      - name: "Fix ownership"
-        run: breeze ci fix-ownership
-        if: always()
-
-  tests-helm-executor-upgrade:
-    timeout-minutes: 150
-    name: Helm Chart Executor Upgrade - ${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}}
-    runs-on: ${{ needs.build-info.outputs.runs-on }}
-    needs: [build-info, wait-for-prod-images]
-    env:
-      RUNS_ON: ${{ needs.build-info.outputs.runs-on }}
-      BACKEND: postgres
-      RUN_TESTS: "true"
-      RUNTIME: "kubernetes"
-      KUBERNETES_MODE: "image"
-      EXECUTOR: "KubernetesExecutor"
-      KIND_VERSION: "${{ needs.build-info.outputs.default-kind-version }}"
-      HELM_VERSION: "${{ needs.build-info.outputs.default-helm-version }}"
-      CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >-
-        ${{needs.build-info.outputs.min-max-python-versions-as-string}}
-      CURRENT_KUBERNETES_VERSIONS_AS_STRING: >-
-        ${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}}
-    if: >
-      needs.build-info.outputs.run-kubernetes-tests == 'true' &&
-      needs.build-info.outputs.default-branch == 'main'
-    steps:
-      - name: Cleanup repo
-        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v3
-        with:
-          persist-credentials: false
-      - name: "Setup python"
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ needs.build-info.outputs.default-python-version }}
-          cache: 'pip'
-          cache-dependency-path: ./dev/breeze/setup*
-      - run: ./scripts/ci/install_breeze.sh
-      - name: "Free space"
-        run: breeze ci free-space
-      - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
-        run: breeze prod-image pull --run-in-parallel --tag-as-latest
+          path: ".build/.k8s-env"
+          key: "k8s-env"
+      - name: K8s Setup env
+        run: breeze k8s setup-env
+      - name: >
+          K8S Build images Python: ${{ needs.build-info.outputs.python-versions-list-as-string }}
+        run: breeze k8s build-k8s-image --run-in-parallel
         env:
           PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
           IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
-      - name: "Cache virtualenv for kubernetes testing"
-        uses: actions/cache@v3
-        with:
-          path: ".build/.kubernetes_venv"
-          key: "kubernetes-${{ needs.build-info.outputs.default-python-version }}\
-  -${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}}
-  -${{needs.build-info.outputs.min-max-python-versions-as-string}}
-  -${{ hashFiles('setup.py','setup.cfg') }}"
-          restore-keys: "kubernetes-${{ needs.build-info.outputs.default-python-version }}-\
-  -${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}} \
-  -${{needs.build-info.outputs.min-max-python-versions-as-string}}"
-      - name: "Cache bin folder with tools for kubernetes testing"
-        uses: actions/cache@v3
-        with:
-          path: ".build/kubernetes-bin"
-          key: "kubernetes-binaries
-  -${{ needs.build-info.outputs.default-kind-version }}\
-  -${{ needs.build-info.outputs.default-helm-version }}"
-          restore-keys: "kubernetes-binaries"
-      - name: "Kubernetes Helm Chart Executor Upgrade Tests"
-        run: ./scripts/ci/kubernetes/ci_upgrade_cluster_with_different_executors_in_parallel.sh
+      - name: K8S Create clusters ${{needs.build-info.outputs.kubernetes-combos}}
+        run: breeze k8s create-cluster --run-in-parallel
         env:
-          PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}"
-      - name: "Upload KinD logs"
+          PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
+          KUBERNETES_VERSIONS: ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
+      - name: K8S Upload images ${{needs.build-info.outputs.kubernetes-combos}}
+        run: breeze k8s upload-k8s-image --run-in-parallel
+        env:
+          PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
+          KUBERNETES_VERSIONS: ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
+          EXECUTOR: ${{matrix.executor}}
+      - name: K8S Configure clusters ${{needs.build-info.outputs.kubernetes-combos}}
+        run: breeze k8s configure-cluster --run-in-parallel
+        env:
+          PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
+          KUBERNETES_VERSIONS: ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
+      - name: K8S Deploy Airflow ${{matrix.executor}} ${{needs.build-info.outputs.kubernetes-combos}}
+        run: breeze k8s deploy-airflow --run-in-parallel
+        env:
+          PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
+          KUBERNETES_VERSIONS: ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
+          EXECUTOR: ${{matrix.executor}}
+      - name: K8S Tests ${{matrix.executor}} ${{needs.build-info.outputs.kubernetes-combos}}
+        run: breeze k8s tests --run-in-parallel
+        env:
+          PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
+          KUBERNETES_VERSIONS: ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
+          EXECUTOR: ${{matrix.executor}}
+      - name: K8S Upgrade ${{matrix.executor}} ${{needs.build-info.outputs.kubernetes-combos}}
+        run: breeze k8s deploy-airflow --run-in-parallel --upgrade --set config.logging.logging_level=INFO
+        env:
+          PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
+          KUBERNETES_VERSIONS: ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
+          EXECUTOR: ${{matrix.executor}}
+      - name: K8S Logs ${{needs.build-info.outputs.kubernetes-combos}}
+        run: breeze k8s logs --all
+        if: failure() || cancelled()
+      - name: Upload KinD logs ${{needs.build-info.outputs.kubernetes-combos}}
         uses: actions/upload-artifact@v3
         if: failure() || cancelled()
         with:
-          name: kind-logs-KubernetesExecutor
+          name: kind-logs-${{matrix.executor}}
           path: /tmp/kind_logs_*
           retention-days: 7
+      - name: "Delete clusters"
+        run: breeze k8s delete-cluster --all
+        if: always()
       - name: "Fix ownership"
         run: breeze ci fix-ownership
         if: always()

.gitpod.yml

@@ -20,7 +20,7 @@
 # Docs: https://www.gitpod.io/docs/config-gitpod-file/
 
 tasks:
-  - init: ./breeze-legacy -y
+  - init: ./script/ci/install_breeze.sh
   - name: Install pre-commit
     openMode: split-right
     command: |

.pre-commit-config.yaml

@@ -84,7 +84,7 @@ repos:
       - id: insert-license
         name: Add license for all shell files
         exclude: ^\.github/.*$|^airflow/_vendor/|^dev/breeze/autocomplete/.*$
-        files: ^breeze-legacy$|^breeze-complete$|\.bash$|\.sh$
+        files: \.bash$|\.sh$
         args:
           - --comment-style
           - "|#|"
@@ -301,7 +301,7 @@ repos:
           The word(s) should be in lowercase." && exec codespell "$@"' --
         language: python
         types: [text]
-        exclude: ^airflow/_vendor/|^RELEASE_NOTES\.txt$|^airflow/www/static/css/material-icons\.css$|^images/.*$
+        exclude: ^airflow/_vendor/|^RELEASE_NOTES\.txt$|^airflow/www/static/css/material-icons\.css$|^images/.*$|^.*package-lock.json$
         args:
           - --ignore-words=docs/spelling_wordlist.txt
           - --skip=docs/*/commits.rst,airflow/providers/*/*.rst,*.lock,INTHEWILD.md,*.min.js,docs/apache-airflow/tutorial/pipeline_example.csv,airflow/www/*.log
@@ -556,12 +556,6 @@ repos:
         files: \.*example_dags.*\.py$
         exclude: ^airflow/_vendor/
         pass_filenames: true
-      - id: check-integrations-are-consistent
-        name: Check if integration list is consistent in various places
-        entry: ./scripts/ci/pre_commit/pre_commit_check_integrations.sh
-        language: system
-        pass_filenames: false
-        files: ^common/_common_values\.sh$|^breeze-complete$
       - id: check-apache-license-rat
         name: Check if licenses are OK for Apache
         entry: ./scripts/ci/pre_commit/pre_commit_check_license.py
@@ -597,16 +591,17 @@ repos:
         require_serial: true
       - id: lint-helm-chart
         name: Lint Helm Chart
-        entry: ./scripts/ci/pre_commit/pre_commit_helm_lint.sh
-        language: system
+        entry: ./scripts/ci/pre_commit/pre_commit_helm_lint.py
+        language: python
         pass_filenames: false
         files: ^chart
         require_serial: true
+        additional_dependencies: ['rich>=12.4.4','requests']
       - id: run-shellcheck
         name: Check Shell scripts syntax correctness
         language: docker_image
         entry: koalaman/shellcheck:v0.8.0 -x -a
-        files: ^breeze-legacy$|^breeze-complete$|\.sh$|^hooks/build$|^hooks/push$|\.bash$
+        files: ^.*\.sh$|^hooks/build$|^hooks/push$|\.bash$
         exclude: ^dev/breeze/autocomplete/.*$
       - id: lint-css
         name: stylelint