[AIRFLOW-6058] Running tests with pytest (apache#6472)

This commit runs Airflow's test suite using pytest.

.gitignore

@@ -157,6 +157,7 @@ airflow-*.err
 airflow-*.out
 airflow-*.log
 airflow-*.pid
+.airflow_db_initialised
 
 # mypy
 .mypy_cache/

.travis.yml

@@ -64,6 +64,7 @@ jobs:
       env: >-
         BACKEND=postgres
         ENV=kubernetes
+        START_KUBERNETES_CLUSTER=true
         KUBERNETES_VERSION=v1.15.0
         KUBERNETES_MODE=persistent_mode
         PYTHON_VERSION=3.6

BREEZE.rst

@@ -617,9 +617,9 @@ This is the current syntax for  `./breeze <./breeze>`_:
   -t, --test-target <TARGET>
           Run the specified unit test target. There might be multiple
           targets specified separated with comas. The <EXTRA_ARGS> passed after -- are treated
-          as additional options passed to nosetest. For example:
+          as additional options passed to pytest. For example:
 
-          './breeze --test-target tests.core -- --logging-level=DEBUG'
+          './breeze --test-target tests/test_core.py -- --logging-level=DEBUG'
 
   -x, --execute-command <COMMAND>
           Run chosen command instead of entering the environment. The command is run using
@@ -725,7 +725,7 @@ Once you run ``./breeze`` you can also execute various actions via generated con
 
    Enter the environment          : ./.build/cmd_run
    Run command in the environment : ./.build/cmd_run "[command with args]" [bash options]
-   Run tests in the environment   : ./.build/test_run [test-target] [nosetest options]
+   Run tests in the environment   : ./.build/test_run [test-target] [pytest options]
    Run Docker compose command     : ./.build/dc [help/pull/...] [docker-compose options]
 
 Troubleshooting

Dockerfile

@@ -233,7 +233,7 @@ RUN mkdir -pv ${AIRFLOW_HOME} \
     mkdir -pv ${AIRFLOW_HOME}/logs
 
 # Increase the value here to force reinstalling Apache Airflow pip dependencies
-ARG PIP_DEPENDENCIES_EPOCH_NUMBER="1"
+ARG PIP_DEPENDENCIES_EPOCH_NUMBER="2"
 ENV PIP_DEPENDENCIES_EPOCH_NUMBER=${PIP_DEPENDENCIES_EPOCH_NUMBER}
 
 # Optimizing installation of Cassandra driver
@@ -330,17 +330,6 @@ RUN npm run prod
 
 COPY ./scripts/docker/entrypoint.sh /entrypoint.sh
 
-COPY .bash_completion run-tests-complete run-tests ${HOME}/
-
-COPY .bash_completion.d/run-tests-complete \
-     ${HOME}/.bash_completion.d/run-tests-complete
-
-RUN echo ". ${HOME}/.bash_completion" >> "${HOME}/.bashrc"
-
-RUN chmod +x "${HOME}/run-tests-complete"
-
-RUN chmod +x "${HOME}/run-tests"
-
 # Copy selected subdirectories only
 COPY .github/ ${AIRFLOW_SOURCES}/.github/
 COPY dags/ ${AIRFLOW_SOURCES}/dags/
@@ -351,7 +340,7 @@ COPY docs/ ${AIRFLOW_SOURCES}/docs/
 COPY tests/ ${AIRFLOW_SOURCES}/tests/
 COPY airflow/ ${AIRFLOW_SOURCES}/airflow/
 COPY .coveragerc .rat-excludes .flake8 pylintrc LICENSE MANIFEST.in NOTICE CHANGELOG.txt \
-     .github .bash_completion .bash_completion.d run-tests run-tests-complete \
+     .github \
      setup.cfg setup.py \
      ${AIRFLOW_SOURCES}/
 

TESTING.rst

@@ -20,7 +20,7 @@
 Airflow Test Infrastructure
 ===========================
 
-* **Unit tests** are Python ``nose`` tests launched with ``run-tests``.
+* **Unit tests** are Python tests that do not require any additional integrations.
   Unit tests are available both in the `Breeze environment <BREEZE.rst>`__
   and local virtualenv.
 
@@ -41,11 +41,7 @@ before tests are executed.
 Airflow Unit Tests
 ==================
 
-All tests for Apache Airflow are run via the ``run-tests`` utility that
-provides a Python testing framework with more than 300
-tests including Unit, Integration and System tests.
-``run-tests`` can be launched as
-a command in the Breeze environment, as a script, or via IDE interface.
+All tests for Apache Airflow are run using `pytest <http://doc.pytest.org/en/latest/>`_ .
 
 Running Unit Tests from IDE
 ---------------------------
@@ -57,116 +53,62 @@ select it as the default project's environment, and run unit tests as follows:
     :align: center
     :alt: Running unit tests
 
-Some of the core tests use dags defined in ``tests/dags`` folder. Those tests should have
-``AIRFLOW__CORE__UNIT_TEST_MODE`` set to True. You can set it up in your test configuration:
-
-.. image:: images/airflow_unit_test_mode.png
-    :align: center
-    :alt: Airflow Unit test mode
-
 Note that you can run the unit tests in the standalone local virtualenv
 (with no Breeze installed) if they do not have dependencies such as
 Postgres/MySQL/Hadoop/etc.
 
-Running Unit Tests from Local virtualenv
-----------------------------------------
 
-You can use the ``run-tests`` script outside the Breeze Docker container,
-directly from your local virtualenv.
-
-In the Breeze environment, the ``run-test`` script is in the path.
-To run it from the local virtualenv, you need to prepend
-it with ``./`` as follows: ``./run-tests``.
-
-This script has several flags that can be useful for your testing.
+Running Unit Tests
+--------------------------------
+To run unit, integration and system tests from the Breeze and your
+virtualenv you can use `pytest <http://doc.pytest.org/en/latest/>`_ framework.
 
-.. code:: text
+Custom pytest plugin run ``airflow db init`` and ``airflow db reset`` the first
+time you launch them, so you can count on the database being initialized. Currently,
+when you run tests not supported **in the local virtualenv, the tests may either fail
+or provide an error message**.
 
-    Usage: run-tests [FLAGS] [TESTS_TO_RUN] -- <EXTRA_NOSETEST_ARGS>
+There are many available options for selecting specific test in pytest. Details could be found
+in official documentation but here are few basic examples:
 
-    Runs tests specified (or all tests if no tests are specified).
+.. code-block:: bash
 
-    Flags:
+    pytest -k "TestCore and not check"
 
-    -h, --help
-            Shows this help message.
+This will run ``TestCore`` class but will skip tests of this class that includes 'check' in their names.
+For better performance (due to test collection) you should do:
 
-    -i, --with-db-init
-            Forces database initialization before tests.
+.. code-block:: bash
 
-    -s, --nocapture
-            Doesn't capture stdout when running the tests. This is useful if you are
-            debugging with ipdb and want to drop into the console with it
-            by adding this line to source code:
+    pytest tests/tests_core.py -k "TestCore and not bash".
 
-                import ipdb; ipdb.set_trace()
+This flag is useful when used like this:
 
-    -v, --verbose
-            Provides verbose output showing coloured output of tests being run and summary
-            of the tests (in a manner similar to the tests run in the CI environment).
+.. code-block:: bash
 
-You can pass extra parameters to ``nose``, by adding ``nose`` arguments after
-``--``. For example, to just execute the "core" unit tests and add ipdb
-``set\_trace`` method, you can run the following command:
+    pytest tests/tests_core.py -k "test_check_operators"
 
-.. code:: bash
+to run single test. This can also be done by specifying full path to the test:
 
-    ./run-tests tests.core:TestCore --nocapture --verbose
+.. code-block:: bash
 
-To add a single test method without colors or debug logs, specify:
+    pytest tests/test_core.py::TestCore::test_check_operators
 
-.. code:: bash
+To run whole test class:
 
-    ./run-tests tests.core:TestCore.test_check_operators
+.. code-block:: bash
 
-Note that the first time the ``./run_tests`` script runs, it
-performs a database initialization. If you run further tests without
-leaving the environment, the database will not be initialized. But you
-can always force the database initialization with the ``--with-db-init``
-(``-i``) switch. The script will inform you what you can do when it is
-run.
+    pytest tests/test_core.py::TestCore
 
 **Note:** We do not provide a clear distinction between tests
 (Unit/Integration/System tests), but we are working on it.
-Currently, when you run tests not supported in the local virtualenv,
-the script may either fail or provide an error message.
-
-Running Unit Tests inside Breeze
---------------------------------
-Tu run unit tests from the Breeze:
-
-1. Enter Airflow Breeze environment.
-
-2. Use ``run-tests``.
-   To pass extra parameters to ``nose``, precede them with '--'.
-
-The tests run ``airflow db reset`` and ``airflow db init`` the first time you
-launch them in a running container, so you can count on the database being initialized.
-All subsequent test executions within the same container will run without database
-initialization.
-You can also optionally add the ``--with-db-init`` flag if you want to re-initialize
-the database.
-
-.. code-block:: bash
-
-   run-tests --with-db-init tests.core:TestCore.test_check_operators -- -s --logging-level=DEBUG
-
-**Examples**:
-
-* Execute the "core" unit tests and pass extra parameters to ``nose``:
-
-  ``run-tests tests.core:TestCore -- -s --logging-level=DEBUG``
-
-* Execute a single test method:
-
-  ``run-tests tests.core:TestCore.test_check_operators -- -s --logging-level=DEBUG``
 
 
 Running Tests for a Specified Target using Breeze from the host
 ---------------------------------------------------------------
 
 If you wish to only run tests and not to drop into shell, you can do this by providing the
-``-t``, ``--test-target`` flag. You can add extra nosetest flags after ``--`` in the command line.
+``-t``, ``--test-target`` flag. You can add extra pytest flags after ``--`` in the command line.
 
 .. code-block:: bash
 
@@ -182,7 +124,7 @@ You can also specify individual tests or a group of tests:
 
 .. code-block:: bash
 
-    ./breeze --test-target tests.core:TestCore
+    ./breeze --test-target tests/test_core.py::TestCore
 
 Running Full test suite via scripts from the host
 -------------------------------------------------

airflow/utils/log/colored_log.py

@@ -89,6 +89,10 @@ def _color_record_traceback(self, record: LogRecord) -> LogRecord:
         return record
 
     def format(self, record: LogRecord) -> str:
-        record = self._color_record_args(record)
-        record = self._color_record_traceback(record)
-        return super().format(record)
+        try:
+            record = self._color_record_args(record)
+            record = self._color_record_traceback(record)
+            return super().format(record)
+        except ValueError:  # I/O operation on closed file
+            from logging import Formatter
+            return Formatter().format(record)

airflow/www/views.py

@@ -1048,7 +1048,6 @@ def blocked(self, session=None):
             for dag_id, active_dag_runs in dags:
                 max_active_runs = 0
                 dag = dagbag.get_dag(dag_id)
-                max_active_runs = dagbag.dags[dag_id].max_active_runs
                 if dag:
                     # TODO: Make max_active_runs a column so we can query for it directly
                     max_active_runs = dag.max_active_runs

breeze

@@ -20,6 +20,8 @@
 # Bash sanity settings (error on exit, complain for undefined vars, error when pipe fails)
 set -euo pipefail
 
+export BREEZE=true
+
 MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 export AIRFLOW_SOURCES="${MY_DIR}"
@@ -332,9 +334,9 @@ The swiss-knife-army tool for Airflow testings. It allows to perform various tes
 -t, --test-target <TARGET>
         Run the specified unit test target. There might be multiple
         targets specified separated with comas. The <EXTRA_ARGS> passed after -- are treated
-        as additional options passed to nosetest. For example:
+        as additional options passed to pytest. For example:
 
-        '${0} --test-target tests.core -- --logging-level=DEBUG'
+        '${0} --test-target tests/test_core.py -- --logging-level=DEBUG'
 
 -x, --execute-command <COMMAND>
         Run chosen command instead of entering the environment. The command is run using
@@ -939,7 +941,7 @@ ${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CONTAINER_BRANCH_NAME}-python${PYT
 printf '=%.0s' $(seq "${SEPARATOR_WIDTH}")
 
 if [[ "${TEST_TARGET}" == "." ]]; then
-    TEST_TARGET=""
+    TEST_TARGET="tests/"
 fi
 
 print_badge