From b4c7538b86094c39a8231fa6d4a3e0f42174c198 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 5 Nov 2019 22:30:00 +0100 Subject: [PATCH] [AIRFLOW-5830] Get rid of slim image (#6494) The slim image gave only very small gain on executing the tests in CI. The image was significantly smaller, but then for local development and testing you needed both full CI and SLIM-CI image. This made the scripts and docker image needlessly complex - especially in the wake of coming Production image it turned to be premature optimisation really. While it sped-up (slightly - by 10-20 seconds) some static check jobs in Travis, it increased time needed by developers to have a working environment and to keep it updated every time it was needed (by minutes) Also having two separately managed images made it rather complex to join some of the Travis CI jobs (there is a follow-up change with getting rid of Checklicence image). With this change both static checks and tests are executed using single image. That also opens doors for further simplification of the scripts and easier implementation of production image. --- .pre-commit-config.yaml | 2 +- BREEZE.rst | 8 +- Dockerfile | 189 ++++++++---------- breeze | 8 +- hooks/build | 78 +------- scripts/ci/_utils.sh | 108 +++------- scripts/ci/ci_docs.sh | 2 +- scripts/ci/ci_flake8.sh | 2 +- scripts/ci/ci_isort.sh | 2 +- scripts/ci/ci_mypy.sh | 2 +- scripts/ci/ci_pylint_main.sh | 2 +- scripts/ci/ci_pylint_tests.sh | 2 +- scripts/ci/ci_refresh_pylint_todo.sh | 2 +- scripts/ci/ci_run_all_static_tests.sh | 4 +- .../ci_run_all_static_tests_except_pylint.sh | 4 +- ..._all_static_tests_except_pylint_licence.sh | 4 +- scripts/ci/ci_run_all_static_tests_pylint.sh | 4 +- .../ci/docker_build/ci_build_install_deps.sh | 68 ------- scripts/ci/in_container/run_pylint_main.sh | 1 + scripts/ci/local_ci_build_ci_slim_image.sh | 2 +- scripts/ci/local_ci_cleanup.sh | 2 - scripts/ci/pre_commit_check_license.sh | 2 +- scripts/ci/pre_commit_ci_build.sh | 25 +++ scripts/ci/pre_commit_flake8.sh | 2 +- scripts/ci/pre_commit_lint_dockerfile.sh | 2 +- scripts/ci/pre_commit_mypy.sh | 2 +- scripts/ci/pre_commit_pylint_main.sh | 2 +- scripts/ci/pre_commit_pylint_tests.sh | 2 +- tests/test_impersonation.py | 4 +- 29 files changed, 169 insertions(+), 368 deletions(-) delete mode 100755 scripts/ci/docker_build/ci_build_install_deps.sh create mode 100755 scripts/ci/pre_commit_ci_build.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9738757cfb773..9be695daae43b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -192,7 +192,7 @@ repos: require_serial: true - id: build name: Check if image build is needed - entry: ./scripts/ci/local_ci_build.sh + entry: ./scripts/ci/pre_commit_ci_build.sh language: system always_run: true pass_filenames: false diff --git a/BREEZE.rst b/BREEZE.rst index 8b77b99a5a512..3df93e66a4623 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -79,11 +79,9 @@ images maintained on the Docker Hub in the ``apache/airflow`` repository. There are three images that we are currently managing: -* **Slim CI** image that is used for static code checks (size of ~500MB). Its tag follows the pattern - of ``-python-ci-slim`` (for example, ``apache/airflow:master-python3.6-ci-slim``). - The image is built using the ``_ Dockerfile. -* **Full CI image*** that is used for testing. It contains a lot more test-related installed software - (size of ~1GB). Its tag follows the pattern of ``-python-ci`` +* **CI image*** that is used for testing od both Unit tests and static check tests. + It contains a lot test-related packages (size of ~1GB). + Its tag follows the pattern of ``-python-ci`` (for example, ``apache/airflow:master-python3.6-ci``). The image is built using the ``_ Dockerfile. * **Checklicense image** that is used during license check with the Apache RAT tool. It does not diff --git a/Dockerfile b/Dockerfile index 28fc424ca335b..3ba54caee9350 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,16 +15,8 @@ # # WARNING: THIS DOCKERFILE IS NOT INTENDED FOR PRODUCTION USE OR DEPLOYMENT. # -# Base image for the whole Docker file -ARG APT_DEPS_IMAGE="airflow-apt-deps-ci-slim" ARG PYTHON_BASE_IMAGE="python:3.6-slim-stretch" -############################################################################################################ -# This is the slim image with APT dependencies needed by Airflow. It is based on a python slim image -# Parameters: -# PYTHON_BASE_IMAGE - base python image (python:x.y-slim-stretch) -############################################################################################################ -FROM ${PYTHON_BASE_IMAGE} as airflow-apt-deps-ci-slim - +FROM ${PYTHON_BASE_IMAGE} as main SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] @@ -121,116 +113,95 @@ RUN adduser airflow \ && echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \ && chmod 0440 /etc/sudoers.d/airflow -############################################################################################################ -# This is an image with all APT dependencies needed by CI. It is built on top of the airlfow APT image -# Parameters: -# airflow-apt-deps - this is the base image for CI deps image. -############################################################################################################ -FROM airflow-apt-deps-ci-slim as airflow-apt-deps-ci +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ -SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] +# Note missing man directories on debian-stretch +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 +RUN mkdir -pv /usr/share/man/man1 \ + && mkdir -pv /usr/share/man/man7 \ + && apt-get update \ + && apt-get install --no-install-recommends -y \ + gnupg \ + apt-transport-https \ + ca-certificates \ + software-properties-common \ + krb5-user \ + ldap-utils \ + less \ + lsb-release \ + net-tools \ + openjdk-8-jdk \ + openssh-client \ + openssh-server \ + postgresql-client \ + python-selinux \ + sqlite3 \ + tmux \ + unzip \ + vim \ + && apt-get autoremove -yqq --purge \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* -ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ +ENV HADOOP_DISTRO="cdh" HADOOP_MAJOR="5" HADOOP_DISTRO_VERSION="5.11.0" HADOOP_VERSION="2.6.0" \ + HADOOP_HOME="/tmp/hadoop-cdh" +ENV HIVE_VERSION="1.1.0" HIVE_HOME="/tmp/hive" +ENV HADOOP_URL="https://archive.cloudera.com/${HADOOP_DISTRO}${HADOOP_MAJOR}/${HADOOP_DISTRO}/${HADOOP_MAJOR}/" +ENV MINICLUSTER_BASE="https://github.com/bolkedebruin/minicluster/releases/download/" \ + MINICLUSTER_HOME="/tmp/minicluster" \ + MINICLUSTER_VER="1.1" -ARG APT_DEPS_IMAGE="airflow-apt-deps-ci-slim" -ENV APT_DEPS_IMAGE=${APT_DEPS_IMAGE} -ARG KUBERNETES_VERSION="v1.15.0" -ENV KUBERNETES_VERSION=${KUBERNETES_VERSION} -ARG KIND_VERSION="v0.5.0" -ENV KIND_VERSION=${KIND_VERSION} +RUN mkdir -pv "${HADOOP_HOME}" \ + && mkdir -pv "${HIVE_HOME}" \ + && mkdir -pv "${MINICLUSTER_HOME}" \ + && mkdir -pv "/user/hive/warehouse" \ + && chmod -R 777 "${HIVE_HOME}" \ + && chmod -R 777 "/user/" -RUN echo "${APT_DEPS_IMAGE}" - -# Note the ifs below might be removed if Buildkit will become usable. It should skip building this -# image automatically if it is not used. For now we still go through all layers below but they are empty -RUN if [[ "${APT_DEPS_IMAGE}" == "airflow-apt-deps-ci" ]]; then \ - # Note missing man directories on debian-stretch - # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 - mkdir -pv /usr/share/man/man1 \ - && mkdir -pv /usr/share/man/man7 \ - && apt-get update \ - && apt-get install --no-install-recommends -y \ - gnupg \ - apt-transport-https \ - ca-certificates \ - software-properties-common \ - krb5-user \ - ldap-utils \ - less \ - lsb-release \ - net-tools \ - openjdk-8-jdk \ - openssh-client \ - openssh-server \ - postgresql-client \ - python-selinux \ - sqlite3 \ - tmux \ - unzip \ - vim \ - && apt-get autoremove -yqq --purge \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - ;\ - fi +ENV HADOOP_DOWNLOAD_URL="${HADOOP_URL}hadoop-${HADOOP_VERSION}-${HADOOP_DISTRO}${HADOOP_DISTRO_VERSION}.tar.gz" \ + HADOOP_TMP_FILE="/tmp/hadoop.tar.gz" -# TODO: We should think about removing those and moving them into docker-compose dependencies. -COPY scripts/ci/docker_build/ci_build_install_deps.sh /tmp/ci_build_install_deps.sh +RUN curl -sL "${HADOOP_DOWNLOAD_URL}" >"${HADOOP_TMP_FILE}" \ + && tar xzf "${HADOOP_TMP_FILE}" --absolute-names --strip-components 1 -C "${HADOOP_HOME}" \ + && rm "${HADOOP_TMP_FILE}" -# Kubernetes dependencies -RUN \ -if [[ "${APT_DEPS_IMAGE}" == "airflow-apt-deps-ci" ]]; then \ - curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ - && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian stretch stable" \ - && apt-get update \ - && apt-get -y install --no-install-recommends docker-ce \ - && apt-get autoremove -yqq --purge \ - && apt-get clean && rm -rf /var/lib/apt/lists/* \ - ;\ -fi +ENV HIVE_URL="${HADOOP_URL}hive-${HIVE_VERSION}-${HADOOP_DISTRO}${HADOOP_DISTRO_VERSION}.tar.gz" \ + HIVE_TMP_FILE="/tmp/hive.tar.gz" -RUN \ -if [[ "${APT_DEPS_IMAGE}" == "airflow-apt-deps-ci" ]]; then \ - curl -Lo kubectl \ - "https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" \ - && chmod +x kubectl \ - && mv kubectl /usr/local/bin/kubectl \ - ;\ -fi +RUN curl -sL "${HIVE_URL}" >"${HIVE_TMP_FILE}" \ + && tar xzf "${HIVE_TMP_FILE}" --strip-components 1 -C "${HIVE_HOME}" \ + && rm "${HIVE_TMP_FILE}" -RUN \ -if [[ "${APT_DEPS_IMAGE}" == "airflow-apt-deps-ci" ]]; then \ - curl -Lo kind \ - "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64" \ - && chmod +x kind \ - && mv kind /usr/local/bin/kind \ - ;\ -fi - -ENV HADOOP_DISTRO=cdh \ - HADOOP_MAJOR=5 \ - HADOOP_DISTRO_VERSION=5.11.0 \ - HADOOP_VERSION=2.6.0 \ - HIVE_VERSION=1.1.0 -ENV HADOOP_URL=https://archive.cloudera.com/${HADOOP_DISTRO}${HADOOP_MAJOR}/${HADOOP_DISTRO}/${HADOOP_MAJOR}/ -ENV HADOOP_HOME=/tmp/hadoop-cdh HIVE_HOME=/tmp/hive - -RUN if [[ "${APT_DEPS_IMAGE}" == "airflow-apt-deps-ci" ]]; then /tmp/ci_build_install_deps.sh; fi +ENV MINICLUSTER_URL="${MINICLUSTER_BASE}${MINICLUSTER_VER}/minicluster-${MINICLUSTER_VER}-SNAPSHOT-bin.zip" \ + MINICLUSTER_TMP_FILE="/tmp/minicluster.zip" + +RUN curl -sL "${MINICLUSTER_URL}" > "${MINICLUSTER_TMP_FILE}" \ + && unzip "${MINICLUSTER_TMP_FILE}" -d "/tmp" \ + && rm "${MINICLUSTER_TMP_FILE}" ENV PATH "${PATH}:/tmp/hive/bin" -############################################################################################################ -# This is the target image - it installs PIP and NPM dependencies including efficient caching -# mechanisms - it might be used to build the bare airflow build or CI build -# Parameters: -# APT_DEPS_IMAGE - image with APT dependencies. It might either be base deps image with airflow -# dependencies or CI deps image that contains also CI-required dependencies -############################################################################################################ -FROM ${APT_DEPS_IMAGE} as main +RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ + && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian stretch stable" \ + && apt-get update \ + && apt-get -y install --no-install-recommends docker-ce \ + && apt-get autoremove -yqq --purge \ + && apt-get clean && rm -rf /var/lib/apt/lists/* -SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] +ARG KUBECTL_VERSION="v1.15.0" +ENV KUBECTL_VERSION=${KUBECTL_VERSION} +ARG KIND_VERSION="v0.5.0" +ENV KIND_VERSION=${KIND_VERSION} -RUN echo "Airflow version: ${AIRFLOW_VERSION}" +RUN curl -Lo kubectl \ + "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \ + && chmod +x kubectl \ + && mv kubectl /usr/local/bin/kubectl + +RUN curl -Lo kind \ + "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64" \ + && chmod +x kind \ + && mv kind /usr/local/bin/kind ARG AIRFLOW_USER=airflow ENV AIRFLOW_USER=${AIRFLOW_USER} @@ -366,15 +337,11 @@ RUN if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \ COPY --chown=airflow:airflow ./scripts/docker/entrypoint.sh /entrypoint.sh -ARG APT_DEPS_IMAGE="airflow-apt-deps-ci-slim" -ENV APT_DEPS_IMAGE=${APT_DEPS_IMAGE} - COPY --chown=airflow:airflow .bash_completion run-tests-complete run-tests ${HOME}/ COPY --chown=airflow:airflow .bash_completion.d/run-tests-complete \ ${HOME}/.bash_completion.d/run-tests-complete -RUN if [[ "${APT_DEPS_IMAGE}" == "airflow-apt-deps-ci" ]]; then \ - ${AIRFLOW_SOURCES}/scripts/ci/docker_build/ci_build_extract_tests.sh; fi +RUN ${AIRFLOW_SOURCES}/scripts/ci/docker_build/ci_build_extract_tests.sh USER ${AIRFLOW_USER} diff --git a/breeze b/breeze index d27cd5dfbafb4..442cb311698cb 100755 --- a/breeze +++ b/breeze @@ -105,10 +105,7 @@ export AIRFLOW_CONTAINER_PUSH_IMAGES=${AIRFLOW_CONTAINER_PUSH_IMAGES:="false"} # For local builds we fix file permissions only for setup-related files export AIRFLOW_FIX_PERMISSIONS=${AIRFLOW_FIX_PERMISSIONS:="setup"} -# Skip building slim image locally - we only need full CI image -export AIRFLOW_CONTAINER_SKIP_CI_SLIM_IMAGE="true" - -# Skip building full CI image locally - we only need slim image +# Skip building full CI image locally export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="false" # Branch name of the base image used (usually master or v1-10-test or v1-10-stable) @@ -682,7 +679,7 @@ echo CMDNAME="$(basename -- "$0")" # Cleans up the answer that was given last time, whether to force/ -cleanup_last_force_answer +forget_last_answer export ENV="${ENV:=$(read_from_file ENV)}" export BACKEND="${BACKEND:=$(read_from_file BACKEND)}" @@ -935,7 +932,6 @@ prepare_command_file "${BUILD_CACHE_DIR}/${LAST_DC_TEST_FILE}" "${DC_RUN_COMMAND prepare_command_file "${BUILD_CACHE_DIR}/${LAST_DC_FILE}" '"' "false" rebuild_ci_image_if_needed -rebuild_ci_slim_image_if_needed rebuild_checklicence_image_if_needed export AIRFLOW_CONTAINER_DOCKER_IMAGE=\ diff --git a/hooks/build b/hooks/build index 6f3b56fd1a630..91e0917c37d80 100755 --- a/hooks/build +++ b/hooks/build @@ -98,30 +98,18 @@ function build_python_image { NAME="${1}" MY_IMAGE_TAG="${2}" TARGET_IMAGE="${3}" - APT_DEPS_IMAGE="${4:-airflow-apt-deps-ci-slim}" - AIRFLOW_EXTRAS="${5:-all}" - AIRFLOW_USER="${6:-airflow}" - HOME="${7:-/home/airflow}" + AIRFLOW_EXTRAS="${4:-all}" + AIRFLOW_USER="${5:-airflow}" + HOME="${6:-/home/airflow}" echo "Build ${NAME} image: ${MY_IMAGE_TAG}" echo "Python base image: ${PYTHON_BASE_IMAGE}" set +u - if [[ "${MY_IMAGE_TAG}" == "${AIRFLOW_CI_IMAGE}" ]]; then - DOCKER_CACHE_DIRECTIVE=("${DOCKER_CACHE_DIRECTIVE_CI[@]}") - elif [[ "${MY_IMAGE_TAG}" == "${AIRFLOW_SLIM_CI_IMAGE}" ]]; then - DOCKER_CACHE_DIRECTIVE=("${DOCKER_CACHE_DIRECTIVE_CI_SLIM[@]}") - else - echo - echo "Don't know how to set cache directive for ${MY_IMAGE_TAG}. Exiting" - echo - exit 1 - fi set -x docker build \ --build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE}" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ - --build-arg APT_DEPS_IMAGE="${APT_DEPS_IMAGE}" \ --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ --build-arg AIRFLOW_USER="${AIRFLOW_USER}" \ --build-arg AIRFLOW_BRANCH="${AIRFLOW_CONTAINER_BRANCH_NAME}" \ @@ -189,9 +177,6 @@ else NON_CI="true" fi -# Extras used to to build slim airflow image -AIRFLOW_SLIM_EXTRAS=${AIRFLOW_SLIM_EXTRAS:="all"} - # Extras used to build cache and CI image AIRFLOW_CI_EXTRAS=${AIRFLOW_CI_EXTRAS:="devel_ci"} @@ -212,13 +197,8 @@ export AIRFLOW_CONTAINER_FORCE_PULL_IMAGES=${AIRFLOW_CONTAINER_FORCE_PULL_IMAGES # In CI environment (and local force) we skip pulling latest python image export AIRFLOW_CONTAINER_SKIP_LATEST_PYTHON_PULL=${AIRFLOW_CONTAINER_SKIP_LATEST_PYTHON_PULL:=${CI}} -# Skips downloading and building the slim image of airflow -# This is set to true by default in CI test environment (we only need full CI image then) -export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE=${AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE:=${CI}} -echo "Skip slim image: ${AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE}" - # Skips downloading and building the CI image, full CI-enabled image of airflow -# This is set to true for pre-tests which do not need the whole CI image just the slim one +# This is set to true for pre-tests which do not need the whole CI image export AIRFLOW_CONTAINER_SKIP_CI_IMAGE=${AIRFLOW_CONTAINER_SKIP_CI_IMAGE:="false"} echo "Skip CI image: ${AIRFLOW_CONTAINER_SKIP_CI_IMAGE}" @@ -241,11 +221,6 @@ export PYTHON_BASE_IMAGE=python:${PYTHON_VERSION}-slim-stretch export CHECKLICENCE_BASE_IMAGE=debian:stretch-slim if [[ "${AIRFLOW_RELEASE_BUILD}" == "true" ]]; then - export AIRFLOW_SLIM_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_VERSION}-python${PYTHON_VERSION}-ci-slim" - export AIRFLOW_SLIM_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_VERSION}-ci-slim" - export AIRFLOW_SLIM_CI_IMAGE_LATEST="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:latest-python${PYTHON_VERSION}-${TAG_PREFIX}-ci-slim" - export AIRFLOW_SLIM_CI_IMAGE_LATEST_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:latest-${TAG_PREFIX}-ci-slim" - export AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_VERSION}-python${PYTHON_VERSION}-ci" export AIRFLOW_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_VERSION}-ci" export AIRFLOW_CI_IMAGE_LATEST="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:latest-python${PYTHON_VERSION}-${TAG_PREFIX}-ci" @@ -253,9 +228,6 @@ if [[ "${AIRFLOW_RELEASE_BUILD}" == "true" ]]; then export AIRFLOW_CHECKLICENCE_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:checklicence" else - export AIRFLOW_SLIM_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${TAG_PREFIX}-python${PYTHON_VERSION}-ci-slim" - export AIRFLOW_SLIM_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${TAG_PREFIX}-ci-slim" - export AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${TAG_PREFIX}-python${PYTHON_VERSION}-ci" export AIRFLOW_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${TAG_PREFIX}-ci" export AIRFLOW_CHECKLICENCE_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:checklicence" @@ -280,7 +252,6 @@ if [[ "${AIRFLOW_CONTAINER_CLEANUP_IMAGES}" == "true" ]]; then start_step "Removing images" docker rmi "${PYTHON_BASE_IMAGE}" || true docker rmi "${CHECKLICENCE_BASE_IMAGE}" || true - docker rmi "${AIRFLOW_SLIM_CI_IMAGE}" || true docker rmi "${AIRFLOW_CI_IMAGE}" || true docker rmi "${AIRFLOW_CHECKLICENCE_IMAGE}" || true echo @@ -297,7 +268,6 @@ fi start_step "Populating cache" DOCKER_CACHE_DIRECTIVE_CI=() -DOCKER_CACHE_DIRECTIVE_CI_SLIM=() DOCKER_CACHE_DIRECTIVE_CHECKLICENCE=() if [[ "${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE}" == "true" ]]; then @@ -328,17 +298,12 @@ if [[ "${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE}" == "true" ]]; then echo "Downloading the CI Airflow image" IMAGES_TO_PULL="${IMAGES_TO_PULL} ${AIRFLOW_CI_IMAGE}" fi - if [[ "${AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE}" != "true" ]]; then - echo "Downloading the slim Airflow image" - IMAGES_TO_PULL="${IMAGES_TO_PULL} ${AIRFLOW_SLIM_CI_IMAGE}" - fi if [[ "${AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE}" != "true" ]]; then echo "Downloading the CHECKLICENCE Airflow image" IMAGES_TO_PULL="${IMAGES_TO_PULL} ${AIRFLOW_CHECKLICENCE_IMAGE}" fi DOCKER_CACHE_DIRECTIVE_CI=() - DOCKER_CACHE_DIRECTIVE_CI_SLIM=() if [[ ${IMAGES_TO_PULL} == "" ]]; then echo echo "Skipping building of all images." @@ -378,8 +343,6 @@ if [[ "${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE}" == "true" ]]; then fi if [[ "${IMAGE}" == "${AIRFLOW_CI_IMAGE}" ]]; then DOCKER_CACHE_DIRECTIVE_CI+=("--cache-from" "${IMAGE}") - elif [[ "${IMAGE}" == "${AIRFLOW_SLIM_CI_IMAGE}" ]]; then - DOCKER_CACHE_DIRECTIVE_CI_SLIM+=("--cache-from" "${IMAGE}") elif [[ "${IMAGE}" == "${AIRFLOW_CHECKLICENCE_IMAGE}" ]]; then DOCKER_CACHE_DIRECTIVE_CHECKLICENCE+=("--cache-from" "${IMAGE}") else @@ -396,14 +359,12 @@ start_step "Setting cache options" if [[ "${AIRFLOW_CONTAINER_USE_NO_CACHE}" == "true" ]]; then DOCKER_CACHE_DIRECTIVE_CI+=("--no-cache") - DOCKER_CACHE_DIRECTIVE_CI_SLIM+=("--no-cache") DOCKER_CACHE_DIRECTIVE_CHECKLICENCE+=("--no-cache") echo echo "Skip cache for builds. Everything will be rebuilt from scratch." echo echo "Cache directives used: " echo "CI build: ${DOCKER_CACHE_DIRECTIVE_CI[*]}" - echo "CI slim build: ${DOCKER_CACHE_DIRECTIVE_CI_SLIM[*]}" echo "CI checklicence build: ${DOCKER_CACHE_DIRECTIVE_CHECKLICENCE[*]}" echo elif [[ "${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE}" == "true" ]]; then @@ -412,13 +373,11 @@ elif [[ "${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE}" == "true" ]]; then echo "Cache directives used: " set +u echo "CI build: ${DOCKER_CACHE_DIRECTIVE_CI[*]}" - echo "CI slim build: ${DOCKER_CACHE_DIRECTIVE_CI_SLIM[*]}" echo "CI checklicence build: ${DOCKER_CACHE_DIRECTIVE_CHECKLICENCE[*]}" set -u echo else DOCKER_CACHE_DIRECTIVE_CI=() - DOCKER_CACHE_DIRECTIVE_CI_SLIM=() DOCKER_CACHE_DIRECTIVE_CHECKLICENCE=() echo echo "Use default cache from locally built images." @@ -426,7 +385,6 @@ else echo "Cache directives used: " set +u echo "CI build: ${DOCKER_CACHE_DIRECTIVE_CI[*]}" - echo "CI slim build: ${DOCKER_CACHE_DIRECTIVE_CI_SLIM[*]}" echo "CI checklicence build: ${DOCKER_CACHE_DIRECTIVE_CHECKLICENCE[*]}" set -u echo @@ -443,7 +401,7 @@ BUILD_ID=${BUILD_ID:="local"} # directory where "deployment" artifacts should be placed -DEPLOY_DIR=${AIRFLOW_SOURCES}/dist/${AIRFLOW_CONTAINER_BRANCH_NAME}/$(date +%Y-%m-%d)/${BUILD_ID}/${PYTHON_VERSION} +DEPLOY_DIR="${AIRFLOW_SOURCES}/dist/${AIRFLOW_CONTAINER_BRANCH_NAME}/$(date "+%Y-%m-%d")/${BUILD_ID}/${PYTHON_VERSION}" mkdir -pv "${DEPLOY_DIR}" @@ -483,29 +441,6 @@ echo "Group/other write access removed for ${FILES_FOR_REBUILD_CHECK[*]}" echo echo -start_step "Build Airflow CI slim image" - -if [[ "${AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE}" == "true" ]]; then - echo "Skip building the Airflow CI slim image" -else - build_python_image "Airflow" \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ - "main" \ - "airflow-apt-deps-ci-slim" - if [[ "${PYTHON_VERSION_FOR_DEFAULT_IMAGE}" == "${PYTHON_VERSION}" ]]; then - docker tag "${AIRFLOW_SLIM_CI_IMAGE}" "${AIRFLOW_SLIM_CI_IMAGE_DEFAULT}" - add_image_to_push "${AIRFLOW_SLIM_CI_IMAGE_DEFAULT}" - fi - if [[ "${AIRFLOW_RELEASE_BUILD}" == "true" ]]; then - docker tag "${AIRFLOW_SLIM_CI_IMAGE}" "${AIRFLOW_SLIM_CI_IMAGE_LATEST}" - add_image_to_push "${AIRFLOW_SLIM_CI_IMAGE_LATEST}" - if [[ "${PYTHON_VERSION_FOR_DEFAULT_IMAGE}" == "${PYTHON_VERSION}" ]]; then - docker tag "${AIRFLOW_SLIM_CI_IMAGE}" "${AIRFLOW_SLIM_CI_IMAGE_LATEST_DEFAULT}" - add_image_to_push "${AIRFLOW_SLIM_CI_IMAGE_LATEST_DEFAULT}" - fi - fi -fi - if [[ "${AIRFLOW_CONTAINER_SKIP_CI_IMAGE}" == "true" ]]; then echo "Skip building the CI full Airflow image" else @@ -513,7 +448,6 @@ else build_python_image "Airflow CI" \ "${AIRFLOW_CI_IMAGE}" \ "main" \ - "airflow-apt-deps-ci" \ "devel_ci" \ "root" \ "/root" @@ -536,8 +470,10 @@ if [[ "${AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE}" == "true" ]]; then echo "Skip building the CHECKLICENCE image" else start_step "Build Airflow CHECKLICENCE image" + set +u docker build . -f Dockerfile-checklicence \ "${DOCKER_CACHE_DIRECTIVE_CHECKLICENCE[@]}" -t "${AIRFLOW_CHECKLICENCE_IMAGE}" + set -u add_image_to_push "${AIRFLOW_CHECKLICENCE_IMAGE}" fi diff --git a/scripts/ci/_utils.sh b/scripts/ci/_utils.sh index 3a12c0c3b4cd7..0daa85027654c 100644 --- a/scripts/ci/_utils.sh +++ b/scripts/ci/_utils.sh @@ -33,7 +33,7 @@ export BUILD_CACHE_DIR LAST_FORCE_ANSWER_FILE="${BUILD_CACHE_DIR}/last_force_answer.sh" -IMAGES_TO_CHECK=("SLIM_CI" "CI" "CHECKLICENCE") +IMAGES_TO_CHECK=("CI" "CHECKLICENCE") export IMAGES_TO_CHECK mkdir -p "${AIRFLOW_SOURCES}/.mypy_cache" @@ -236,11 +236,8 @@ function update_all_md5_files() { # or the Dockerfile itself changes. # # Another reason to skip rebuilding Docker is thar currently it takes a bit longer time than simple Docker -# files. There are the following, problems with the current Dockerfiles that need longer build times: -# 1) We need to fix group permissions of files in Docker because different linux build services have -# different default umask and Docker uses group permissions in checking for cache invalidation. -# 2) we use multi-stage build and in case of slim image we needlessly build a full CI image because -# support for this only comes with the upcoming buildkit: https://github.com/docker/cli/issues/1134 +# We need to fix group permissions of files in Docker because different linux build services have +# different default umask and Docker uses group permissions in checking for cache invalidation. # # As result of this check - most of the static checks will start pretty much immediately. # @@ -355,29 +352,29 @@ function assert_not_in_container() { fi } -function cleanup_last_force_answer() { - # Removes the "Force answer" (yes/no/quit) given previously, unles you specifically want to use it. +function forget_last_answer() { + # Removes the "Forced answer" (yes/no/quit) given previously, unles you specifically want to remember it. # # This is the default behaviour of all rebuild scripts to ask independently whether you want to - # rebuild the image or not. Sometimes however we want to reuse answer previously given. For + # rebuild the image or not. Sometimes however we want to remember answer previously given. For # example if you answered "no" to rebuild the image, the assumption is that you do not - # want to rebuild image for other rebuilds in the same pre-commit execution. + # want to rebuild image also for other rebuilds in the same pre-commit execution. # - # All the pre-commit checks therefore have `export SKIP_CLEANUP_OF_LAST_ANSWER="true"` set + # All the pre-commit checks therefore have `export REMEMBER_LAST_ANSWER="true"` set # So that in case they are run in a sequence of commits they will not rebuild. Similarly if your most # recent answer was "no" and you run `pre-commit run mypy` (for example) it will also reuse the # "no" answer given previously. This happens until you run any of the breeze commands or run all # precommits `pre-commit run` - then the "LAST_FORCE_ANSWER_FILE" will be removed and you will # be asked again. - if [[ ${SKIP_CLEANUP_OF_LAST_ANSWER:=""} != "true" ]]; then + if [[ ${REMEMBER_LAST_ANSWER:="false"} != "true" ]]; then print_info - print_info "Removing last answer from ${LAST_FORCE_ANSWER_FILE}" + print_info "Forgetting last answer from ${LAST_FORCE_ANSWER_FILE}:" print_info rm -f "${LAST_FORCE_ANSWER_FILE}" else if [[ -f "${LAST_FORCE_ANSWER_FILE}" ]]; then print_info - print_info "Retaining last answer from ${LAST_FORCE_ANSWER_FILE}" + print_info "Still remember last answer from ${LAST_FORCE_ANSWER_FILE}:" print_info "$(cat "${LAST_FORCE_ANSWER_FILE}")" print_info fi @@ -493,47 +490,10 @@ EOF fi } -# -# Rebuilds the slim image for static checks if needed. In order to speed it up, it's built without NPM -# -function rebuild_ci_slim_image_if_needed() { - export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE="false" - export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="true" - export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE="true" - - # Temporary force python version 3.5 for static checks - export OLD_PYTHON_VERSION=${PYTHON_VERSION=""} - export PYTHON_VERSION=3.5 - - export THE_IMAGE_TYPE="SLIM_CI" - - rebuild_image_if_needed - - export AIRFLOW_SLIM_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${DEFAULT_BRANCH}-python${PYTHON_VERSION}-ci-slim" - export PYTHON_VERSION=${OLD_PYTHON_VERSION} -} - -# -# Cleans up the CI slim image -# -function cleanup_ci_slim_image() { - export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE="false" - export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="true" - export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE="true" - export AIRFLOW_CONTAINER_CLEANUP_IMAGES="true" - - export PYTHON_VERSION=3.5 # Always use python version 3.5 for static checks - - export THE_IMAGE_TYPE="SLIM_CI" - - rebuild_image_if_needed -} - # # Rebuilds the image for tests if needed. # function rebuild_ci_image_if_needed() { - export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE="true" export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE="true" export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="false" @@ -546,10 +506,9 @@ function rebuild_ci_image_if_needed() { # -# Cleans up the CI slim image +# Cleans up the CI image # function cleanup_ci_image() { - export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE="true" export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="false" export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE="true" export AIRFLOW_CONTAINER_CLEANUP_IMAGES="true" @@ -563,7 +522,6 @@ function cleanup_ci_image() { # Rebuilds the image for licence checks if needed. # function rebuild_checklicence_image_if_needed() { - export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE="true" export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE="false" export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="true" @@ -575,10 +533,9 @@ function rebuild_checklicence_image_if_needed() { } # -# Cleans up the CI slim image +# Cleans up the checklicence image # function cleanup_checklicence_image() { - export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE="true" export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="true" export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE="false" export AIRFLOW_CONTAINER_CLEANUP_IMAGES="true" @@ -648,7 +605,7 @@ function basic_sanity_checks() { go_to_airflow_sources check_if_coreutils_installed create_cache_directory - cleanup_last_force_answer + forget_last_answer } @@ -664,7 +621,7 @@ function run_flake8() { --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ + "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" \ | tee -a "${OUTPUT_LOG}" else @@ -676,7 +633,7 @@ function run_flake8() { --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ + "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" "${FILES[@]}" \ | tee -a "${OUTPUT_LOG}" fi @@ -691,7 +648,7 @@ function run_docs() { --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ + "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/docs/build.sh" \ | tee -a "${OUTPUT_LOG}" } @@ -721,7 +678,7 @@ function run_mypy() { --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ + "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/ci/in_container/run_mypy.sh" "airflow" "tests" "docs" \ | tee -a "${OUTPUT_LOG}" else @@ -733,7 +690,7 @@ function run_mypy() { --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ + "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/ci/in_container/run_mypy.sh" "${FILES[@]}" \ | tee -a "${OUTPUT_LOG}" fi @@ -750,7 +707,7 @@ function run_pylint_main() { --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ + "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" \ | tee -a "${OUTPUT_LOG}" else @@ -762,7 +719,7 @@ function run_pylint_main() { --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ + "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" "${FILES[@]}" \ | tee -a "${OUTPUT_LOG}" fi @@ -780,7 +737,7 @@ function run_pylint_tests() { --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ + "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" \ | tee -a "${OUTPUT_LOG}" else @@ -792,7 +749,7 @@ function run_pylint_tests() { --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ + "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" "${FILES[@]}" \ | tee -a "${OUTPUT_LOG}" fi @@ -852,7 +809,7 @@ function refresh_pylint_todo() { --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ - "${AIRFLOW_SLIM_CI_IMAGE}" | tee -a "${OUTPUT_LOG}" + "${AIRFLOW_CI_IMAGE}" | tee -a "${OUTPUT_LOG}" } function rebuild_all_images_if_needed_and_confirmed() { @@ -861,15 +818,7 @@ function rebuild_all_images_if_needed_and_confirmed() { for THE_IMAGE_TYPE in "${IMAGES_TO_CHECK[@]}" do - if [[ "${THE_IMAGE_TYPE}" == "SLIM_CI" ]]; then - # Temporary force python version 3.5 for static checks - export OLD_PYTHON_VERSION=${PYTHON_VERSION=""} - export PYTHON_VERSION=3.5 - - check_if_docker_build_is_needed - - export PYTHON_VERSION=${OLD_PYTHON_VERSION} - fi + check_if_docker_build_is_needed done if [[ ${AIRFLOW_CONTAINER_DOCKER_BUILD_NEEDED} == "true" ]]; then @@ -901,7 +850,6 @@ function rebuild_all_images_if_needed_and_confirmed() { if [[ ${SKIP_REBUILD} != "true" ]]; then rebuild_ci_image_if_needed - rebuild_ci_slim_image_if_needed rebuild_checklicence_image_if_needed fi fi @@ -969,16 +917,16 @@ function build_image_on_ci() { elif [[ ${TRAVIS_JOB_NAME} == "Check lic"* ]]; then rebuild_checklicence_image_if_needed elif [[ ${TRAVIS_JOB_NAME} == "Static"* ]]; then - rebuild_ci_slim_image_if_needed + rebuild_ci_image_if_needed elif [[ ${TRAVIS_JOB_NAME} == "Pylint"* ]]; then match_files_regexp '.*\.py' if [[ ${FILE_MATCHES} == "true" || ${TRAVIS_PULL_REQUEST:=} == "false" ]]; then - rebuild_ci_slim_image_if_needed + rebuild_ci_image_if_needed else touch "${BUILD_CACHE_DIR}"/.skip_tests fi elif [[ ${TRAVIS_JOB_NAME} == "Build documentation"* ]]; then - rebuild_ci_slim_image_if_needed + rebuild_ci_image_if_needed else echo echo "Error! Unexpected Travis job name: ${TRAVIS_JOB_NAME}" diff --git a/scripts/ci/ci_docs.sh b/scripts/ci/ci_docs.sh index 603f19b86811c..bc6c48cffd7cd 100755 --- a/scripts/ci/ci_docs.sh +++ b/scripts/ci/ci_docs.sh @@ -31,7 +31,7 @@ basic_sanity_checks script_start -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed run_docs diff --git a/scripts/ci/ci_flake8.sh b/scripts/ci/ci_flake8.sh index 958305171b9f8..553043090db61 100755 --- a/scripts/ci/ci_flake8.sh +++ b/scripts/ci/ci_flake8.sh @@ -31,7 +31,7 @@ basic_sanity_checks script_start -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed run_flake8 "$@" diff --git a/scripts/ci/ci_isort.sh b/scripts/ci/ci_isort.sh index a8ada59956858..01e7b1163da31 100755 --- a/scripts/ci/ci_isort.sh +++ b/scripts/ci/ci_isort.sh @@ -32,7 +32,7 @@ force_python_3_5 script_start -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed run_flake8 "$@" diff --git a/scripts/ci/ci_mypy.sh b/scripts/ci/ci_mypy.sh index 6f0837c83a82b..8d4dd22b4cb17 100755 --- a/scripts/ci/ci_mypy.sh +++ b/scripts/ci/ci_mypy.sh @@ -31,7 +31,7 @@ basic_sanity_checks script_start -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed run_mypy "$@" diff --git a/scripts/ci/ci_pylint_main.sh b/scripts/ci/ci_pylint_main.sh index 8c2d533047801..430effd0d2b83 100755 --- a/scripts/ci/ci_pylint_main.sh +++ b/scripts/ci/ci_pylint_main.sh @@ -31,7 +31,7 @@ basic_sanity_checks script_start -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed if [[ "${#@}" != "0" ]]; then filter_out_files_from_pylint_todo_list "$@" diff --git a/scripts/ci/ci_pylint_tests.sh b/scripts/ci/ci_pylint_tests.sh index 2edc53f5ebd26..df7c189929700 100755 --- a/scripts/ci/ci_pylint_tests.sh +++ b/scripts/ci/ci_pylint_tests.sh @@ -31,7 +31,7 @@ basic_sanity_checks script_start -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed if [[ "${#@}" != "0" ]]; then filter_out_files_from_pylint_todo_list "$@" diff --git a/scripts/ci/ci_refresh_pylint_todo.sh b/scripts/ci/ci_refresh_pylint_todo.sh index bbeed2c843ab6..46716c3150884 100755 --- a/scripts/ci/ci_refresh_pylint_todo.sh +++ b/scripts/ci/ci_refresh_pylint_todo.sh @@ -30,7 +30,7 @@ basic_sanity_checks script_start -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed refresh_pylint_todo diff --git a/scripts/ci/ci_run_all_static_tests.sh b/scripts/ci/ci_run_all_static_tests.sh index 6e80267556d34..03ae9aca5057c 100755 --- a/scripts/ci/ci_run_all_static_tests.sh +++ b/scripts/ci/ci_run_all_static_tests.sh @@ -39,10 +39,10 @@ if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then exit fi -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed rebuild_checklicence_image_if_needed -IMAGES_TO_CHECK=("SLIM_CI" "CHECKLICENCE") +IMAGES_TO_CHECK=("CI" "CHECKLICENCE") export IMAGES_TO_CHECK pre-commit run --all-files --show-diff-on-failure diff --git a/scripts/ci/ci_run_all_static_tests_except_pylint.sh b/scripts/ci/ci_run_all_static_tests_except_pylint.sh index ec26b0679ba4f..e6943b79b2111 100755 --- a/scripts/ci/ci_run_all_static_tests_except_pylint.sh +++ b/scripts/ci/ci_run_all_static_tests_except_pylint.sh @@ -39,10 +39,10 @@ if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then exit fi -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed rebuild_checklicence_image_if_needed -IMAGES_TO_CHECK=("SLIM_CI" "CHECKLICENCE") +IMAGES_TO_CHECK=("CI" "CHECKLICENCE") export IMAGES_TO_CHECK SKIP=pylint pre-commit run --all-files --show-diff-on-failure diff --git a/scripts/ci/ci_run_all_static_tests_except_pylint_licence.sh b/scripts/ci/ci_run_all_static_tests_except_pylint_licence.sh index 55e71b1ac60aa..0899eb6210222 100755 --- a/scripts/ci/ci_run_all_static_tests_except_pylint_licence.sh +++ b/scripts/ci/ci_run_all_static_tests_except_pylint_licence.sh @@ -39,9 +39,9 @@ if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then exit fi -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed -IMAGES_TO_CHECK=("SLIM_CI") +IMAGES_TO_CHECK=("CI") export IMAGES_TO_CHECK SKIP=pylint,check-apache-license pre-commit run --all-files --show-diff-on-failure diff --git a/scripts/ci/ci_run_all_static_tests_pylint.sh b/scripts/ci/ci_run_all_static_tests_pylint.sh index 2b9fd03fb6667..5bb8a655e552e 100755 --- a/scripts/ci/ci_run_all_static_tests_pylint.sh +++ b/scripts/ci/ci_run_all_static_tests_pylint.sh @@ -39,9 +39,9 @@ if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then exit fi -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed -IMAGES_TO_CHECK=("SLIM_CI") +IMAGES_TO_CHECK=("CI") export IMAGES_TO_CHECK pre-commit run pylint --all-files --show-diff-on-failure diff --git a/scripts/ci/docker_build/ci_build_install_deps.sh b/scripts/ci/docker_build/ci_build_install_deps.sh deleted file mode 100755 index eb690822872e3..0000000000000 --- a/scripts/ci/docker_build/ci_build_install_deps.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Bash sanity settings (error on exit, complain for undefined vars, error when pipe fails) -set -euxo pipefail - -# TODO: We should think about removing those and moving them into docker-compose dependencies. -# TODO: We might come up with just one airflow CI image not the SLIM/CI versions. That would simplify a lot. -# TODO: We could likely get rid of the multi-staging approach. It introduces a number of limitations -# TODO: As long as we decrease the size of the CI image, we should be fine with using single CI image for -# TODO: everything. Currently the CI image is about 1GB where CI_SLIM is around 0.5 GB. - -export HADOOP_DISTRO="${HADOOP_DISTRO:="cdh"}" -export HADOOP_MAJOR="${HADOOP_MAJOR:="5"}" -export HADOOP_DISTRO_VERSION="${HADOOP_DISTRO_VERSION:="5.11.0"}" -export HADOOP_VERSION="${HADOOP_VERSION:="2.6.0"}" -export HIVE_VERSION="${HIVE_VERSION:="1.1.0"}" -export HADOOP_URL="${HADOOP_URL:="https://archive.cloudera.com/${HADOOP_DISTRO}${HADOOP_MAJOR}/${HADOOP_DISTRO}/${HADOOP_MAJOR}/"}" -export HADOOP_HOME="${HADOOP_HOME:="/tmp/hadoop-cdh"}" -export HIVE_HOME="${HIVE_HOME:="/tmp/hive"}" -export MINICLUSTER_BASE="${MINICLUSTER_BASE:="https://github.com/bolkedebruin/minicluster/releases/download/"}" -export MINICLUSTER_HOME="${MINICLUSTER_HOME:="/tmp/minicluster"}" -export MINICLUSTER_VER="${MINICLUSTER_VER:="1.1"}" - -mkdir -pv "${HADOOP_HOME}" -mkdir -pv "${HIVE_HOME}" -mkdir -pv "${MINICLUSTER_HOME}" -mkdir -pv "/user/hive/warehouse" -chmod -R 777 "${HIVE_HOME}" -chmod -R 777 "/user/" - -export HADOOP_DOWNLOAD_URL="${HADOOP_URL}hadoop-${HADOOP_VERSION}-${HADOOP_DISTRO}${HADOOP_DISTRO_VERSION}.tar.gz" -export HADOOP_TMP_FILE="/tmp/hadoop.tar.gz" - -curl -sL "${HADOOP_DOWNLOAD_URL}" >"${HADOOP_TMP_FILE}" - -tar xzf "${HADOOP_TMP_FILE}" --absolute-names --strip-components 1 -C "${HADOOP_HOME}" - -rm "${HADOOP_TMP_FILE}" - -export HIVE_URL="${HADOOP_URL}hive-${HIVE_VERSION}-${HADOOP_DISTRO}${HADOOP_DISTRO_VERSION}.tar.gz" -export HIVE_TMP_FILE="/tmp/hive.tar.gz" - -curl -sL "${HIVE_URL}" >"${HIVE_TMP_FILE}" -tar xzf "${HIVE_TMP_FILE}" --strip-components 1 -C "${HIVE_HOME}" -rm "${HIVE_TMP_FILE}" - -MINICLUSTER_URL="${MINICLUSTER_BASE}${MINICLUSTER_VER}/minicluster-${MINICLUSTER_VER}-SNAPSHOT-bin.zip" -MINICLUSTER_TMP_FILE="/tmp/minicluster.zip" - -curl -sL "${MINICLUSTER_URL}" > "${MINICLUSTER_TMP_FILE}" -unzip "${MINICLUSTER_TMP_FILE}" -d "/tmp" -rm "${MINICLUSTER_TMP_FILE}" diff --git a/scripts/ci/in_container/run_pylint_main.sh b/scripts/ci/in_container/run_pylint_main.sh index f8935c973e704..198636ca8d1f1 100755 --- a/scripts/ci/in_container/run_pylint_main.sh +++ b/scripts/ci/in_container/run_pylint_main.sh @@ -41,6 +41,7 @@ if [[ ${#@} == "0" ]]; then -path "./airflow/www/node_modules" -prune -o \ -path "./airflow/www_rbac/node_modules" -prune -o \ -path "./airflow/_vendor" -prune -o \ + -path "./airflow/migrations/versions" -prune -o \ -path "./.eggs" -prune -o \ -path "./docs/_build" -prune -o \ -path "./build" -prune -o \ diff --git a/scripts/ci/local_ci_build_ci_slim_image.sh b/scripts/ci/local_ci_build_ci_slim_image.sh index 6c0635551b42a..b632f0009e2cf 100755 --- a/scripts/ci/local_ci_build_ci_slim_image.sh +++ b/scripts/ci/local_ci_build_ci_slim_image.sh @@ -30,6 +30,6 @@ basic_sanity_checks script_start -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed script_end diff --git a/scripts/ci/local_ci_cleanup.sh b/scripts/ci/local_ci_cleanup.sh index 388731b9619bd..9600c5d3b5609 100755 --- a/scripts/ci/local_ci_cleanup.sh +++ b/scripts/ci/local_ci_cleanup.sh @@ -32,8 +32,6 @@ script_start cleanup_ci_image -cleanup_ci_slim_image - cleanup_checklicence_image script_end diff --git a/scripts/ci/pre_commit_check_license.sh b/scripts/ci/pre_commit_check_license.sh index da38ea7fb50bd..fbcff1497cb7e 100755 --- a/scripts/ci/pre_commit_check_license.sh +++ b/scripts/ci/pre_commit_check_license.sh @@ -20,6 +20,6 @@ set -uo pipefail MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"} -export SKIP_CLEANUP_OF_LAST_ANSWER="true" +export REMEMBER_LAST_ANSWER="true" "${MY_DIR}/ci_check_license.sh" "${@}" diff --git a/scripts/ci/pre_commit_ci_build.sh b/scripts/ci/pre_commit_ci_build.sh new file mode 100755 index 0000000000000..217434ba77240 --- /dev/null +++ b/scripts/ci/pre_commit_ci_build.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -uo pipefail + +MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +export REMEMBER_LAST_ANSWER="false" +export PYTHON_VERSION=3.5 + +"${MY_DIR}/local_ci_build.sh" diff --git a/scripts/ci/pre_commit_flake8.sh b/scripts/ci/pre_commit_flake8.sh index c8fc2f6c6d1af..5108751d5d39e 100755 --- a/scripts/ci/pre_commit_flake8.sh +++ b/scripts/ci/pre_commit_flake8.sh @@ -20,6 +20,6 @@ set -uo pipefail MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"} -export SKIP_CLEANUP_OF_LAST_ANSWER="true" +export REMEMBER_LAST_ANSWER="true" "${MY_DIR}/ci_flake8.sh" "${@}" diff --git a/scripts/ci/pre_commit_lint_dockerfile.sh b/scripts/ci/pre_commit_lint_dockerfile.sh index e742a6d672471..bb729c114f6f6 100755 --- a/scripts/ci/pre_commit_lint_dockerfile.sh +++ b/scripts/ci/pre_commit_lint_dockerfile.sh @@ -19,6 +19,6 @@ set -uo pipefail MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -export SKIP_CLEANUP_OF_LAST_ANSWER="true" +export REMEMBER_LAST_ANSWER="true" "${MY_DIR}/ci_lint_dockerfile.sh" "${@}" diff --git a/scripts/ci/pre_commit_mypy.sh b/scripts/ci/pre_commit_mypy.sh index 116a410ebec2f..d137d20fce9c0 100755 --- a/scripts/ci/pre_commit_mypy.sh +++ b/scripts/ci/pre_commit_mypy.sh @@ -20,6 +20,6 @@ set -uo pipefail MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"} -export SKIP_CLEANUP_OF_LAST_ANSWER="true" +export REMEMBER_LAST_ANSWER="true" "${MY_DIR}/ci_mypy.sh" "${@}" diff --git a/scripts/ci/pre_commit_pylint_main.sh b/scripts/ci/pre_commit_pylint_main.sh index 58b48e1c7af27..70f496fe91fbc 100755 --- a/scripts/ci/pre_commit_pylint_main.sh +++ b/scripts/ci/pre_commit_pylint_main.sh @@ -20,6 +20,6 @@ set -uo pipefail MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"} -export SKIP_CLEANUP_OF_LAST_ANSWER="true" +export REMEMBER_LAST_ANSWER="true" "${MY_DIR}/ci_pylint_main.sh" "${@}" diff --git a/scripts/ci/pre_commit_pylint_tests.sh b/scripts/ci/pre_commit_pylint_tests.sh index 7ff27ac65bd1d..dcb7386118e60 100755 --- a/scripts/ci/pre_commit_pylint_tests.sh +++ b/scripts/ci/pre_commit_pylint_tests.sh @@ -20,6 +20,6 @@ set -uo pipefail MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"} -export SKIP_CLEANUP_OF_LAST_ANSWER="true" +export REMEMBER_LAST_ANSWER="true" "${MY_DIR}/ci_pylint_tests.sh" "${@}" diff --git a/tests/test_impersonation.py b/tests/test_impersonation.py index cc723d9bd480d..c3013e9436ed9 100644 --- a/tests/test_impersonation.py +++ b/tests/test_impersonation.py @@ -80,11 +80,11 @@ def revoke_permissions(): def check_original_docker_image(): - if not os.path.isfile('/.dockerenv') or os.environ.get('APT_DEPS_IMAGE') is None: + if not os.path.isfile('/.dockerenv') or os.environ.get('PYTHON_BASE_IMAGE') is None: raise unittest.SkipTest("""Adding/removing a user as part of a test is very bad for host os (especially if the user already existed to begin with on the OS), therefore we check if we run inside a the official docker container and only allow to run the test there. This is done by checking /.dockerenv -file (always present inside container) and checking for APT_DEPS_IMAGE variable. +file (always present inside container) and checking for PYTHON_BASE_IMAGE variable. """)