[AIRFLOW-5826] Added auto-generation of breeze output in BREEZE.rst (a…

…pache#6490) This is needed to keep breeze --help in sync with the documentation. It makes it easier for the follow-up changes needed for production image to keep the docs in sync with the code.
wendy-wong · Nov 4, 2019 · d2595de · d2595de
1 parent bb01616
commit d2595de
Show file tree

Hide file tree

Showing 6 changed files with 197 additions and 147 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -190,6 +190,13 @@ repos:
         entry: "./scripts/ci/pre_commit_lint_dockerfile.sh"
         files: ^Dockerfile.*$
         pass_filenames: true
+      - id: update-breeze-file
+        name: Update output of breeze command in BREEZE.rst
+        entry: "./scripts/ci/pre_commit_breeze_cmd_line.sh"
+        language: system
+        files: ^BREEZE.rst$|^breeze$|^breeze-complete$
+        pass_filenames: false
+        require_serial: true
       - id: mypy
         name: Run mypy
         language: system

diff --git a/BREEZE.rst b/BREEZE.rst
@@ -839,167 +839,185 @@ Airflow Breeze Syntax
 
 This is the current syntax for  `./breeze <./breeze>`_:
 
+ .. START BREEZE HELP MARKER
+
 .. code-block:: text
 
-    Usage: breeze [FLAGS] \
-      [-k]|[-S <STATIC_CHECK>]|[-F <STATIC_CHECK>]|[-O]|[-e]|[-a]|[-b]|[-t <TARGET>]|[-x <COMMAND>]|[-d <COMMAND>] \
-      -- <EXTRA_ARGS>
 
-    Commands
 
-      By default, the ``breeze`` script enters an IT environment and drops you to a bash shell,
-      but you can also choose commands to run specific actions instead:
+  Usage: breeze [FLAGS] \
+    [-k]|[-S <STATIC_CHECK>]|[-F <STATIC_CHECK>]|[-O]|[-e]|[-a]|[-b]|[-t <TARGET>]|[-x <COMMAND>]|[-d <COMMAND>] \
+    -- <EXTRA_ARGS>
+
+  The swiss-knife-army tool for Airflow testings. It allows to perform various test tasks:
+
+    * Enter interactive environment when no command flags are specified (default behaviour)
+    * Stop the interactive environment with -k, --stop-environment command
+    * Run static checks - either for currently staged change or for all files with
+      -S, --static-check or -F, --static-check-all-files commanbd
+    * Build documentation with -O, --build-docs command
+    * Setup local virtualenv with -e, --setup-virtualenv command
+    * Setup autocomplete for itself with -a, --setup-autocomplete command
+    * Build docker image with -b, --build-only command
+    * Run test target specified with -t, --test-target connad
+    * Execute arbitrary command in the test environmenrt with -x, --execute-command command
+    * Execute arbitrary docker-compose command with -d, --docker-compose command
+
+  ** Commands
+
+    By default the script enters IT environment and drops you to bash shell,
+    but you can also choose one of the commands to run specific actions instead:
+
+  -k, --stop-environment
+          Bring down running docker compose environment. When you start the environment, the docker
+          containers will continue running so that startup time is shorter. But they take quite a lot of
+          memory and CPU. This command stops all running containers from the environment.
+
+  -O, --build-docs
+         Build documentation.
+
+  -S, --static-check <STATIC_CHECK>
+          Run selected static checks for currently changed files. You should specify static check that
+          you would like to run or 'all' to run all checks. One of
+          [ all all-but-pylint check-hooks-apply check-merge-conflict check-executables-have-shebangs check-xml detect-private-key doctoc end-of-file-fixer flake8 forbid-tabs insert-license check-apache-license lint-dockerfile mixed-line-ending mypy pylint shellcheck].
+          You can pass extra arguments including options to to the pre-commit framework as
+          <EXTRA_ARGS> passed after --. For example:
 
-    -k, --stop-environment
-            Stops running a Docker Compose environment. When you start the environment, the Docker
-            containers continue running so that startup time is shorter. But they take quite a lot of
-            memory and CPU. This command stops all running containers in the environment.
+          './breeze  --static-check mypy' or
+          './breeze  --static-check mypy -- --files tests/core.py'
 
-    -O, --build-docs
-           Builds documentation.
+          You can see all the options by adding --help EXTRA_ARG:
 
-    -S, --static-check <STATIC_CHECK>
-            Runs selected static checks for currently changed files. Specify a static check that
-            you would like to run or use 'all' to run all checks. One of
-            [ all all-but-pylint check-hooks-apply check-merge-conflict check-executables-have-shebangs
-              check-xml detect-private-key doctoc end-of-file-fixer flake8 forbid-tabs insert-license
-              check-apache-license lint-dockerfile mixed-line-ending mypy pylint shellcheck].
-            You can pass extra arguments including options to the pre-commit framework as
-            <EXTRA_ARGS> passed after --. For example:
+          './breeze  --static-check mypy -- --help'
 
-            './breeze  --static-check mypy' or
-            './breeze  --static-check mypy -- --files tests/core.py'
+  -F, --static-check-all-files <STATIC_CHECK>
+          Run selected static checks for all applicable files. You should specify static check that
+          you would like to run or 'all' to run all checks. One of
+          [ all all-but-pylint check-hooks-apply check-merge-conflict check-executables-have-shebangs check-xml detect-private-key doctoc end-of-file-fixer flake8 forbid-tabs insert-license check-apache-license lint-dockerfile mixed-line-ending mypy pylint shellcheck].
+          You can pass extra arguments including options to the pre-commit framework as
+          <EXTRA_ARGS> passed after --. For example:
 
-            You can see all the options by adding --help EXTRA_ARG:
+          './breeze --static-check-all-files mypy' or
+          './breeze --static-check-all-files mypy -- --verbose'
 
-            './breeze  --static-check mypy -- --help'
+          You can see all the options by adding --help EXTRA_ARG:
 
-    -F, --static-check-all-files <STATIC_CHECK>
-            Runs selected static checks for all applicable files. Specify a static check that
-            you would like to run or use 'all' to run all checks. One of
-            [ all all-but-pylint check-hooks-apply check-merge-conflict check-executables-have-shebangs
-              check-xml detect-private-key doctoc end-of-file-fixer flake8 forbid-tabs insert-license
-              check-apache-license lint-dockerfile mixed-line-ending mypy pylint shellcheck].
-            You can pass extra arguments including options to the pre-commit framework as
-            <EXTRA_ARGS> passed after --. For example:
+          './breeze --static-check-all-files mypy -- --help'
 
-            './breeze --static-check-all-files mypy' or
-            './breeze --static-check-all-files mypy -- --verbose'
+  -e, --initialize-local-virtualenv
+          Initializes locally created virtualenv installing all dependencies of Airflow.
+          This local virtualenv can be used to aid autocompletion and IDE support as
+          well as run unit tests directly from the IDE. You need to have virtualenv
+          activated before running this command.
 
-            You can see all the options by adding --help EXTRA_ARG:
+  -a, --setup-autocomplete
+          Sets up autocomplete for breeze commands. Once you do it you need to re-enter the bash
+          shell and when typing breeze command <TAB> will provide autocomplete for parameters and values.
 
-            './breeze --static-check-all-files mypy -- --help'
+  -b, --build-only
+          Only build docker images but do not enter the airflow-testing docker container.
 
-    -e, --initialize-local-virtualenv
-            Initializes a locally created virtualenv installing all dependencies of Airflow.
-            This local virtualenv can be used to aid autocompletion and IDE support as
-            well as run unit tests directly from the IDE. You need to have virtualenv
-            activated before running this command.
+  -t, --test-target <TARGET>
+          Run the specified unit test target. There might be multiple
+          targets specified separated with comas. The <EXTRA_ARGS> passed after -- are treated
+          as additional options passed to nosetest. For example:
 
-    -a, --setup-autocomplete
-            Sets up autocompletion for breeze commands. Once you do it, you need to re-enter the bash
-            shell. When you type the breeze command, <TAB> will autocomplete parameters and values.
+          './breeze --test-target tests.core -- --logging-level=DEBUG'
 
-    -b, --build-only
-            Only builds Docker images but does not enter the airflow-testing Docker container.
+  -x, --execute-command <COMMAND>
+          Run chosen command instead of entering the environment. The command is run using
+          'bash -c "<command with args>" if you need to pass arguments to your command, you need
+          to pass them together with command surrounded with " or '. Alternatively you can pass arguments as
+           <EXTRA_ARGS> passed after --. For example:
 
-    -t, --test-target <TARGET>
-            Runs the specified unit test target. You can specify multiple
-            targets separated with commas. The <EXTRA_ARGS> passed after -- are treated
-            as additional options passed to nosetest. For example:
+          './breeze --execute-command "ls -la"' or
+          './breeze --execute-command ls -- --la'
 
-            './breeze --test-target tests.core -- --logging-level=DEBUG'
+  -d, --docker-compose <COMMAND>
+          Run docker-compose command instead of entering the environment. Use 'help' command
+          to see available commands. The <EXTRA_ARGS> passed after -- are treated
+          as additional options passed to docker-compose. For example
 
-    -x, --execute-command <COMMAND>
-            Runs the specified command instead of entering the environment. The command is run using
-            'bash -c "<command with args>". If you need to pass arguments to your command, you need
-            to pass them together with the command surrounded with " or '. Alternatively, you can pass
-            arguments as <EXTRA_ARGS> passed after --. For example:
+          './breeze --docker-compose pull -- --ignore-pull-failures'
 
-            './breeze --execute-command "ls -la"' or
-            './breeze --execute-command ls -- --la'
+  ** General flags
 
-    -d, --docker-compose <COMMAND>
-            Runs the docker-compose command instead of entering the environment. Use the 'help' command
-            to see available commands. The <EXTRA_ARGS> passed after -- are treated
-            as additional options passed to docker-compose. For example:
+  -h, --help
+          Shows this help message.
 
-            './breeze --docker-compose pull -- --ignore-pull-failures'
+  -P, --python <PYTHON_VERSION>
+          Python version used for the image. This is always major/minor version.
+          One of [ 3.5 3.6 3.7 ]. Default is the python3 or python on the path.
 
-    ** General flags
+  -E, --env <ENVIRONMENT>
+          Environment to use for tests. It determines which types of tests can be run.
+          One of [ docker kubernetes ]. Default: docker
 
-    -h, --help
-            Shows this help message.
+  -B, --backend <BACKEND>
+          Backend to use for tests - it determines which database is used.
+          One of [ sqlite mysql postgres ]. Default: sqlite
 
-    -P, --python <PYTHON_VERSION>
-            Specifies a Python version for the image. This is always major/minor version.
-            One of [ 3.5 3.6 3.7 ]. Default is the python3 or python on the path.
+  -K, --kubernetes-version <KUBERNETES_VERSION>
+          Kubernetes version - only used in case of 'kubernetes' environment.
+          One of [ v1.13.0 ]. Default: v1.13.0
 
-    -E, --env <ENVIRONMENT>
-            Specifies an environment for tests. The environment determines which types of tests can be run.
-            One of [ docker kubernetes ]. Default: docker.
+  -M, --kubernetes-mode <KUBERNETES_MODE>
+          Kubernetes mode - only used in case of 'kubernetes' environment.
+          One of [ persistent_mode git_mode ]. Default: git_mode
 
-    -B, --backend <BACKEND>
-            Specifies backend for tests. It determines which database is used.
-            One of [ sqlite mysql postgres ]. Default: sqlite.
+  -s, --skip-mounting-source-volume
+          Skips mounting local volume with sources - you get exactly what is in the
+          docker image rather than your current local sources of airflow.
 
-    -K, --kubernetes-version <KUBERNETES_VERSION>
-            Specifies Kubernetes version. The flag is applicable if the 'kubernetes' environment is used.
-            One of [ v1.13.0 ]. Default: v1.13.0.
+  -v, --verbose
+          Show verbose information about executed commands (enabled by default for running test)
 
-    -M, --kubernetes-mode <KUBERNETES_MODE>
-            Specifies Kubernetes mode. The flag is applicable if the 'kubernetes' environment is used.
-            One of [ persistent_mode git_mode ]. Default: git_mode.
+  -y, --assume-yes
+          Assume 'yes' answer to all questions.
 
-    -s, --skip-mounting-source-volume
-            Skips mounting local volume with sources. You get exactly what is in the
-            Docker image rather than your current local sources of Airflow.
+  -n, --assume-no
+          Assume 'no' answer to all questions.
 
-    -v, --verbose
-            Shows verbose information about executed commands (enabled by default for running tests).
+  -C, --toggle-suppress-cheatsheet
+          Toggles on/off cheatsheet displayed before starting bash shell
 
-    -y, --assume-yes
-            Assumes 'yes' answer to all questions.
+  -A, --toggle-suppress-asciiart
+          Toggles on/off asciiart displayed before starting bash shell
 
-    -n, --assume-no
-            Assumes 'no' answer to all questions.
+  ** Dockerfile management flags
 
-    -C, --toggle-suppress-cheatsheet
-            Toggles on/off the cheatsheet displayed before starting bash shell.
+  -D, --dockerhub-user
+          DockerHub user used to pull, push and build images. Default: apache.
 
-    -A, --toggle-suppress-asciiart
-            Toggles on/off asciiart displayed before starting bash shell.
+  -H, --dockerhub-repo
+          DockerHub repository used to pull, push, build images. Default: airflow.
 
-    ** Dockerfile management flags
+  -r, --force-build-images
+          Forces building of the local docker images. The images are rebuilt
+          automatically for the first time or when changes are detected in
+          package-related files, but you can force it using this flag.
 
-    -D, --dockerhub-user
-            Specifies a Docker Hub user that pulls, pushes and builds images. Default: apache.
+  -R, --force-build-images-clean
+          Force build images without cache. This will remove the pulled or build images
+          and start building images from scratch. This might take a long time.
 
-    -H, --dockerhub-repo
-            Specifies a Docker Hub repository used to pull, push, and build images. Default: airflow.
+  -p, --force-pull-images
+          Forces pulling of images from DockerHub before building to populate cache. The
+          images are pulled by default only for the first time you run the
+          environment, later the locally build images are used as cache.
 
-    -r, --force-build-images
-            Forces building the local Docker images. The images are rebuilt
-            automatically for the first time or when changes are detected in
-            package-related files, but you can force it using this flag.
+  -u, --push-images
+          After building - uploads the images to DockerHub
+          It is useful in case you use your own DockerHub user to store images and you want
+          to build them locally. Note that you need to use 'docker login' before you upload images.
 
-    -R, --force-build-images-clean
-            Forces building images without cache. This removes the pulled or built images
-            and starts building images from scratch. This may take time.
+  -c, --cleanup-images
+          Cleanup your local docker cache of the airflow docker images. This will not reclaim space in
+          docker cache. You need to 'docker system prune' (optionally with --all) to reclaim that space.
 
-    -p, --force-pull-images
-            Forces pulling images from Docker Hub before building to populate the cache. The
-            images are pulled by default only for the first time you run the
-            environment. Later the locally built images are used from the cache.
 
-    -u, --push-images
-            Uploads the images to Docker Hub after rebuilding.
-            It is useful in case you use your own Docker Hub user to store images and you want
-            to build them locally. Note that you need to use 'docker login' before you upload images.
+ .. END BREEZE HELP MARKER
 
-    -c, --cleanup-images
-            Cleans up your local Docker cache of the Airflow Docker images. This does not reclaim space in the
-            Docker cache. You need to 'docker system prune' (optionally with --all) to reclaim that space.
 
 Convenience Scripts
 -------------------

diff --git a/common/_autodetect_variables.sh b/common/_autodetect_variables.sh
@@ -55,13 +55,9 @@ export DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"}
 
 # if AIRFLOW_CONTAINER_BRANCH_NAME is not set it will be set to either SOURCE_BRANCH
 # (if overridden by configuration) or default branch configured in /common/_default_branch.sh
-SOURCE_BRANCH=${SOURCE_BRANCH:=${DEFAULT_BRANCH}}
+export SOURCE_BRANCH=${SOURCE_BRANCH:=${DEFAULT_BRANCH}}
 
-AIRFLOW_CONTAINER_BRANCH_NAME=${AIRFLOW_CONTAINER_BRANCH_NAME:=${SOURCE_BRANCH}}
-
-echo
-echo "Branch: ${AIRFLOW_CONTAINER_BRANCH_NAME}"
-echo
+export AIRFLOW_CONTAINER_BRANCH_NAME=${AIRFLOW_CONTAINER_BRANCH_NAME:=${SOURCE_BRANCH}}
 
 # IMAGE_NAME might come from DockerHub and we decide which PYTHON_VERSION to use based on it (see below)
 # In case IMAGE_NAME is not set we determine PYTHON_VERSION based on the default python on the path
@@ -71,15 +67,8 @@ echo
 # to a different CI system and start pushing images to DockerHub rather than build it there.
 export BASE_IMAGE_NAME=${IMAGE_NAME:=${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CONTAINER_BRANCH_NAME}-python${PYTHON_VERSION:-3.5}}
 
-echo
-echo "BASE_IMAGE_NAME=${BASE_IMAGE_NAME:=} (final image will have -ci, -ci-slim suffixes)"
-echo
-
 # Remove index.docker.io/ prefix as it is added by default by DockerHub
 export LOCAL_BASE_IMAGE_NAME=${BASE_IMAGE_NAME#index.docker.io/}
-echo
-echo "LOCAL_BASE_IMAGE_NAME=${LOCAL_BASE_IMAGE_NAME}"
-echo
 
 if [[ ! ${LOCAL_BASE_IMAGE_NAME} == ${DOCKERHUB_USER}/${DOCKERHUB_REPO}* ]]; then
     echo >&2
@@ -89,16 +78,10 @@ if [[ ! ${LOCAL_BASE_IMAGE_NAME} == ${DOCKERHUB_USER}/${DOCKERHUB_REPO}* ]]; the
 fi
 
 # Extract IMAGE_TAG from LOCAL_BASE_IMAGE_NAME (apache/airflow:master-python3.6 -> master-python3.6)
-IMAGE_TAG=${LOCAL_BASE_IMAGE_NAME##${DOCKERHUB_USER}/${DOCKERHUB_REPO}:}
-echo
-echo "Image tag: ${IMAGE_TAG}"
-echo
+export IMAGE_TAG=${LOCAL_BASE_IMAGE_NAME##${DOCKERHUB_USER}/${DOCKERHUB_REPO}:}
 
 # Extract TAG_PREFIX from IMAGE_TAG (master-python3.6 -> master)
-TAG_PREFIX=${IMAGE_TAG%-*}
-echo
-echo "Image tag prefix: ${TAG_PREFIX}"
-echo
+export TAG_PREFIX=${IMAGE_TAG%-*}
 
 # By that time no matter if we got the version from the IMAGE_NAME or we built
 # IMAGE_NAME with PYTHON_VERSION - the LOCAL_BASE_IMAGE_NAME should end with the correct pythonX.Y
@@ -107,10 +90,6 @@ echo
 # will produce LONG_PYTHON_VERSION_FROM_IMAGE=python3.6
 export LONG_PYTHON_VERSION_FROM_IMAGE="${LOCAL_BASE_IMAGE_NAME##*-}"
 
-echo
-echo "LONG_PYTHON_VERSION_FROM_IMAGE=${LONG_PYTHON_VERSION_FROM_IMAGE}"
-echo
-
 # Verify that python version retrieved from image follows the expected pattern of python3.Y
 if [[ ! ${LONG_PYTHON_VERSION_FROM_IMAGE} =~ python[23]\.[0-9]+ ]]; then
     echo >&2
@@ -123,7 +102,3 @@ fi
 
 # Unless python_version is forced, read actual python version from IMAGE NAME
 export PYTHON_VERSION=${LONG_PYTHON_VERSION_FROM_IMAGE#python}
-
-echo
-echo "PYTHON_VERSION=${PYTHON_VERSION}"
-echo
diff --git a/scripts/ci/ci_before_install.sh b/scripts/ci/ci_before_install.sh
@@ -31,6 +31,8 @@ build_image_on_ci
 
 KUBERNETES_VERSION=${KUBERNETES_VERSION:=""}
 
+mkdir -p "${AIRFLOW_SOURCES}/files"
+
 sudo pip install pre-commit
 
 script_end