Revert "[AIRFLOW-5704] Improve Kind Kubernetes scripts for local test…

…ing (apache#6496)"

This reverts commit 8e789a3.

.gitignore

@@ -142,8 +142,8 @@ rat-results.txt
 # Kubernetes generated templated files
 *.generated
 *.tar.gz
-scripts/ci/in_container/kubernetes/kube/.generated/airflow.yaml
-scripts/ci/in_container/kubernetes/docker/requirements.txt
+scripts/ci/kubernetes/kube/.generated/airflow.yaml
+scripts/ci/kubernetes/docker/requirements.txt
 
 # Node & Webpack Stuff
 *.entry.js
@@ -171,7 +171,5 @@ dmypy.json
 /hive_scratch_dir/
 /.bash_aliases
 /.bash_history
-/.kube
 /.inputrc
 log.txt*
-/.docker_image.tar

.travis.yml

@@ -44,43 +44,44 @@ jobs:
     - name: "Tests postgres kubernetes python 3.6 (persistent)"
       env: >-
         BACKEND=postgres
-        PYTHON_VERSION=3.6
-        START_KUBERNETES_CLUSTER=true
+        ENV=kubernetes
+        KUBERNETES_VERSION=v1.15.0
         KUBERNETES_MODE=persistent_mode
-        KUBERNETES_VERSION=v1.15.3
+        PYTHON_VERSION=3.6
       python: "3.6"
       stage: test
-      script: travis_wait 30 "./scripts/ci/ci_run_airflow_testing.sh" tests.integration.kubernetes
     - name: "Tests postgres kubernetes python 3.6 (git)"
       env: >-
         BACKEND=postgres
-        PYTHON_VERSION=3.6
-        START_KUBERNETES_CLUSTER=true
+        ENV=kubernetes
+        KUBERNETES_VERSION=v1.15.0
         KUBERNETES_MODE=git_mode
-        KUBERNETES_VERSION=v1.15.3
+        PYTHON_VERSION=3.6
       python: "3.6"
       stage: test
-      script: travis_wait 30 "./scripts/ci/ci_run_airflow_testing.sh" tests.integration.kubernetes
     - name: "Tests postgres python 3.6"
       env: >-
         BACKEND=postgres
+        ENV=docker
         PYTHON_VERSION=3.6
       python: "3.6"
       stage: test
     - name: "Tests sqlite python 3.5"
       env:
         BACKEND=sqlite
+        ENV=docker
         PYTHON_VERSION=3.5
       python: "3.5"
       stage: test
     - name: "Tests mysql python 3.7"
       env:
         BACKEND=mysql
+        ENV=docker
         PYTHON_VERSION=3.7
       python: "3.7"
       stage: test
 services:
   - docker
 before_install:
   - ./scripts/ci/ci_before_install.sh
-script: "./scripts/ci/ci_run_airflow_testing.sh"
+script: travis_wait 30 "./scripts/ci/ci_run_airflow_testing.sh"

BREEZE.rst

@@ -77,13 +77,13 @@ Docker Images Used by Breeze
 For all development tasks, related integration tests and static code checks, we use Docker
 images maintained on the Docker Hub in the ``apache/airflow`` repository.
 
-There are those images that we are currently managing:
+There are three images that we are currently managing:
 
-* **CI image*** that is used for testing (both static tests and unit/integration tests).
-  It contains a lot test-related installed software (size of ~1GB).
-  It's tag follows the pattern of ``<BRANCH>-python<PYTHON_VERSION>-ci``
-  (for example, ``apache/airflow:master-python3.6-ci``). 
-  The image is built using the `<Dockerfile>`_ Dockerfile.
+* **CI image*** that is used for testing od both Unit tests and static check tests.
+  It contains a lot test-related packages (size of ~1GB).
+  Its tag follows the pattern of ``<BRANCH>-python<PYTHON_VERSION>-ci``
+  (for example, ``apache/airflow:master-python3.6-ci``). The image is built using the
+  `<Dockerfile>`_ Dockerfile.
 
 Before you run tests, enter the environment or run local static checks, the necessary local images should be
 pulled and built from Docker Hub. This happens automatically for the test environment but you need to
@@ -224,7 +224,7 @@ follows:
 
   .. code-block:: bash
 
-    ./breeze --python 3.6 --backend mysql
+    ./breeze --python 3.6 --backend mysql --env docker
 
 The choices you make are persisted in the ``./.build/`` cache directory so that next time when you use the
 ``breeze`` script, it could use the values that were used previously. This way you do not have to specify
@@ -233,6 +233,25 @@ default settings.
 
 The defaults when you run the Breeze environment are Python 3.6, Sqlite, and Docker.
 
+Available Docker Environments
+..............................
+
+You can choose a container environment when you run Breeze with ``--env`` flag.
+Running the default ``docker`` environment takes a considerable amount of resources. You can run a
+slimmed-down version of the environment - just the Apache Airflow container - by choosing ``bare``
+environment instead.
+
+The following environments are available:
+
+ * The ``docker`` environment (default): starts all dependencies required by a full integration test suite
+   (Postgres, Mysql, Celery, etc). This option is resource intensive so do not forget to
+   [stop environment](#stopping-the-environment) when you are finished. This option is also RAM intensive
+   and can slow down your machine.
+ * The ``kubernetes`` environment: Runs Airflow tests within a Kubernetes cluster.
+ * The ``bare`` environment:  runs Airflow in the Docker without any external dependencies.
+   It only works for independent tests. You can only run it with the sqlite backend.
+
+
 Cleaning Up the Environment
 ---------------------------
 
@@ -516,7 +535,6 @@ the database.
 
    run-tests --with-db-init tests.core:TestCore.test_check_operators -- -s --logging-level=DEBUG
 
-
 Running Tests for a Specified Target
 ------------------------------------
 
@@ -539,17 +557,6 @@ You can also specify individual tests or a group of tests:
 
     ./breeze --test-target tests.core:TestCore
 
-Running Tests with Kubernetes Cluster
--------------------------------------
-
-If you want to run Kubernetes tests, you need to set use ``--start-kubernetes-cluster`` Breeze flag.
-It will create Kubernetes cluster (using `kind (Kubernetes-IN-Docker) <https://kind.sigs.k8s.io/>`_.)
-right after entering the container.
-This might take some time. Note that in order to use latest sources you need to
-rebuild the latest docker image for Breeze first.
-
-All kubernetes tests are in ``tests.integration.kubernetes`` package.
-
 Running Static Code Checks
 --------------------------
 
@@ -710,43 +717,33 @@ On the host:
   ./scripts/ci/ci_pylint.sh ./airflow/example_dags/test_utils.py
 
 Running Test Suites via Scripts
--------------------------------
-
-To run tests use ``./scripts/ci/local_ci_run_airflow_testing.sh`` scripts.
-
-Optionally use one of the variables (defaults are bold):
-
-* PYTHON_VERSION is one of **3.5**/3.6/3.7
-* BACKEND is one of **sqlite**/postgres/mysql
-
-For Kubernetes tests you should set ``START_KUBERNETES_CLUSTER`` variable to true.
-This will start Kubernetes cluster inside the container
-using `kind (Kubernetes-IN-Docker) <https://kind.sigs.k8s.io/>`_.
-You might also optionally specify one of the variables (defaults are bold):
+--------------------------------------------
 
-* KUBERNETES_MODE is a mode of kubernetes: one of persistent_mode/**git_mode**
-* KUBERNETES_VERSION is required for Kubernetes tests. Default: v1.15.3.
-
-To run all non-kubernetes tests with default settings (Python 3.6, Sqlite backend, no kubernetes), enter:
+To run all tests with default settings (Python 3.6, Sqlite backend, "docker" environment), enter:
 
 .. code-block::
 
   ./scripts/ci/local_ci_run_airflow_testing.sh
 
 
-To select Python 3.5 version, Postgres backend, specify:
+To select Python 3.5 version, Postgres backend, and a "docker" environment, specify:
 
 .. code-block::
 
-  PYTHON_VERSION=3.5 BACKEND=postgres ./scripts/ci/local_ci_run_airflow_testing.sh
+  PYTHON_VERSION=3.5 BACKEND=postgres ENV=docker ./scripts/ci/local_ci_run_airflow_testing.sh
 
 To run Kubernetes tests, enter:
 
 .. code-block::
 
-  START_KUBERNETES_CLUSTER=true KUBERNETES_MODE=persistent_mode KUBERNETES_VERSION=v1.15.3 BACKEND=postgres \
+  KUBERNETES_VERSION==v1.13.0 KUBERNETES_MODE=persistent_mode BACKEND=postgres ENV=kubernetes \
     ./scripts/ci/local_ci_run_airflow_testing.sh
 
+* PYTHON_VERSION is one of 3.5/3.6/3.7
+* BACKEND is one of postgres/sqlite/mysql
+* ENV is one of docker/kubernetes/bare
+* KUBERNETES_VERSION is required for Kubernetes tests. Currently, it is KUBERNETES_VERSION=v1.13.0.
+* KUBERNETES_MODE is a mode of kubernetes: either persistent_mode or git_mode.
 
 Using Your Host IDE with Breeze
 ===============================
@@ -945,29 +942,22 @@ This is the current syntax for  `./breeze <./breeze>`_:
           Python version used for the image. This is always major/minor version.
           One of [ 3.5 3.6 3.7 ]. Default is the python3 or python on the path.
 
+  -E, --env <ENVIRONMENT>
+          Environment to use for tests. It determines which types of tests can be run.
+          One of [ docker kubernetes ]. Default: docker
+
   -B, --backend <BACKEND>
           Backend to use for tests - it determines which database is used.
           One of [ sqlite mysql postgres ]. Default: sqlite
 
-  -K, --start-kubernetes-cluster
-          Starts test kubernetes cluster after entering the environment. The cluster is started using
-          Kubernete Mode selected and Kubernetes version specifed via --kubernetes-mode and
-          --kubernetes-version flags.
-
-  -Z, --recreate-kubernetes-cluster
-          Recreates kubernetes cluster if one has already been created. By default if you do not stop
-          environment, the kubernetes cluster created for testing is continuously running and when
-          you start kubernetes testing again it will be reused. You can force deletion and recreation
-          of such cluster with this flag.
+  -K, --kubernetes-version <KUBERNETES_VERSION>
+          Kubernetes version - only used in case of 'kubernetes' environment.
+          One of [ v1.13.0 ]. Default: v1.13.0
 
   -M, --kubernetes-mode <KUBERNETES_MODE>
-          Kubernetes mode - only used in case --start-kubernetes-cluster flag is specified.
+          Kubernetes mode - only used in case of 'kubernetes' environment.
           One of [ persistent_mode git_mode ]. Default: git_mode
 
-  -V, --kubernetes-version <KUBERNETES_VERSION>
-          Kubernetes version - only used in case --start-kubernetes-cluster flag is specified.
-          One of [ v1.15.3 v1.16.2 ]. Default: v1.15.3
-
   -s, --skip-mounting-source-volume
           Skips mounting local volume with sources - you get exactly what is in the
           docker image rather than your current local sources of airflow.
@@ -1021,6 +1011,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
 
  .. END BREEZE HELP MARKER
 
+
 Convenience Scripts
 -------------------
 

Dockerfile

@@ -188,7 +188,7 @@ RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \
     && apt-get autoremove -yqq --purge \
     && apt-get clean && rm -rf /var/lib/apt/lists/*
 
-ARG KUBECTL_VERSION="v1.15.3"
+ARG KUBECTL_VERSION="v1.15.0"
 ENV KUBECTL_VERSION=${KUBECTL_VERSION}
 ARG KIND_VERSION="v0.5.0"
 ENV KIND_VERSION=${KIND_VERSION}
@@ -214,7 +214,7 @@ ENV RAT_JAR_MD5="${RAT_JAR}.md5" \
 RUN echo "Downloading RAT from ${RAT_URL} to ${RAT_JAR}" \
     && curl -sL "${RAT_URL}" > "${RAT_JAR}" \
     && curl -sL "${RAT_URL_MD5}" > "${RAT_JAR_MD5}" \
-    && jar -tf "${RAT_JAR}" >/dev/null \
+    && jar -tf "${RAT_JAR}" \
     && md5sum -c <<<"$(cat "${RAT_JAR_MD5}") ${RAT_JAR}"
 
 ARG AIRFLOW_USER=airflow
@@ -324,6 +324,7 @@ COPY --chown=airflow:airflow airflow/bin/airflow ${AIRFLOW_SOURCES}/airflow/bin/
 # In non-CI optimized build this will install all dependencies before installing sources.
 RUN pip install -e ".[${AIRFLOW_EXTRAS}]"
 
+
 WORKDIR ${AIRFLOW_SOURCES}/airflow/www
 
 # Copy all www files here so that we can run npm building for production