diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9738757cfb773..9be695daae43b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -192,7 +192,7 @@ repos: require_serial: true - id: build name: Check if image build is needed - entry: ./scripts/ci/local_ci_build.sh + entry: ./scripts/ci/pre_commit_ci_build.sh language: system always_run: true pass_filenames: false diff --git a/BREEZE.rst b/BREEZE.rst index 8b77b99a5a512..3df93e66a4623 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -79,11 +79,9 @@ images maintained on the Docker Hub in the ``apache/airflow`` repository. There are three images that we are currently managing: -* **Slim CI** image that is used for static code checks (size of ~500MB). Its tag follows the pattern - of ``-python-ci-slim`` (for example, ``apache/airflow:master-python3.6-ci-slim``). - The image is built using the ``_ Dockerfile. -* **Full CI image*** that is used for testing. It contains a lot more test-related installed software - (size of ~1GB). Its tag follows the pattern of ``-python-ci`` +* **CI image*** that is used for testing od both Unit tests and static check tests. + It contains a lot test-related packages (size of ~1GB). + Its tag follows the pattern of ``-python-ci`` (for example, ``apache/airflow:master-python3.6-ci``). The image is built using the ``_ Dockerfile. * **Checklicense image** that is used during license check with the Apache RAT tool. It does not diff --git a/Dockerfile b/Dockerfile index 28fc424ca335b..3ba54caee9350 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,16 +15,8 @@ # # WARNING: THIS DOCKERFILE IS NOT INTENDED FOR PRODUCTION USE OR DEPLOYMENT. # -# Base image for the whole Docker file -ARG APT_DEPS_IMAGE="airflow-apt-deps-ci-slim" ARG PYTHON_BASE_IMAGE="python:3.6-slim-stretch" -############################################################################################################ -# This is the slim image with APT dependencies needed by Airflow. It is based on a python slim image -# Parameters: -# PYTHON_BASE_IMAGE - base python image (python:x.y-slim-stretch) -############################################################################################################ -FROM ${PYTHON_BASE_IMAGE} as airflow-apt-deps-ci-slim - +FROM ${PYTHON_BASE_IMAGE} as main SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] @@ -121,116 +113,95 @@ RUN adduser airflow \ && echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \ && chmod 0440 /etc/sudoers.d/airflow -############################################################################################################ -# This is an image with all APT dependencies needed by CI. It is built on top of the airlfow APT image -# Parameters: -# airflow-apt-deps - this is the base image for CI deps image. -############################################################################################################ -FROM airflow-apt-deps-ci-slim as airflow-apt-deps-ci +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ -SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] +# Note missing man directories on debian-stretch +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 +RUN mkdir -pv /usr/share/man/man1 \ + && mkdir -pv /usr/share/man/man7 \ + && apt-get update \ + && apt-get install --no-install-recommends -y \ + gnupg \ + apt-transport-https \ + ca-certificates \ + software-properties-common \ + krb5-user \ + ldap-utils \ + less \ + lsb-release \ + net-tools \ + openjdk-8-jdk \ + openssh-client \ + openssh-server \ + postgresql-client \ + python-selinux \ + sqlite3 \ + tmux \ + unzip \ + vim \ + && apt-get autoremove -yqq --purge \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* -ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ +ENV HADOOP_DISTRO="cdh" HADOOP_MAJOR="5" HADOOP_DISTRO_VERSION="5.11.0" HADOOP_VERSION="2.6.0" \ + HADOOP_HOME="/tmp/hadoop-cdh" +ENV HIVE_VERSION="1.1.0" HIVE_HOME="/tmp/hive" +ENV HADOOP_URL="https://archive.cloudera.com/${HADOOP_DISTRO}${HADOOP_MAJOR}/${HADOOP_DISTRO}/${HADOOP_MAJOR}/" +ENV MINICLUSTER_BASE="https://github.com/bolkedebruin/minicluster/releases/download/" \ + MINICLUSTER_HOME="/tmp/minicluster" \ + MINICLUSTER_VER="1.1" -ARG APT_DEPS_IMAGE="airflow-apt-deps-ci-slim" -ENV APT_DEPS_IMAGE=${APT_DEPS_IMAGE} -ARG KUBERNETES_VERSION="v1.15.0" -ENV KUBERNETES_VERSION=${KUBERNETES_VERSION} -ARG KIND_VERSION="v0.5.0" -ENV KIND_VERSION=${KIND_VERSION} +RUN mkdir -pv "${HADOOP_HOME}" \ + && mkdir -pv "${HIVE_HOME}" \ + && mkdir -pv "${MINICLUSTER_HOME}" \ + && mkdir -pv "/user/hive/warehouse" \ + && chmod -R 777 "${HIVE_HOME}" \ + && chmod -R 777 "/user/" -RUN echo "${APT_DEPS_IMAGE}" - -# Note the ifs below might be removed if Buildkit will become usable. It should skip building this -# image automatically if it is not used. For now we still go through all layers below but they are empty -RUN if [[ "${APT_DEPS_IMAGE}" == "airflow-apt-deps-ci" ]]; then \ - # Note missing man directories on debian-stretch - # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 - mkdir -pv /usr/share/man/man1 \ - && mkdir -pv /usr/share/man/man7 \ - && apt-get update \ - && apt-get install --no-install-recommends -y \ - gnupg \ - apt-transport-https \ - ca-certificates \ - software-properties-common \ - krb5-user \ - ldap-utils \ - less \ - lsb-release \ - net-tools \ - openjdk-8-jdk \ - openssh-client \ - openssh-server \ - postgresql-client \ - python-selinux \ - sqlite3 \ - tmux \ - unzip \ - vim \ - && apt-get autoremove -yqq --purge \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - ;\ - fi +ENV HADOOP_DOWNLOAD_URL="${HADOOP_URL}hadoop-${HADOOP_VERSION}-${HADOOP_DISTRO}${HADOOP_DISTRO_VERSION}.tar.gz" \ + HADOOP_TMP_FILE="/tmp/hadoop.tar.gz" -# TODO: We should think about removing those and moving them into docker-compose dependencies. -COPY scripts/ci/docker_build/ci_build_install_deps.sh /tmp/ci_build_install_deps.sh +RUN curl -sL "${HADOOP_DOWNLOAD_URL}" >"${HADOOP_TMP_FILE}" \ + && tar xzf "${HADOOP_TMP_FILE}" --absolute-names --strip-components 1 -C "${HADOOP_HOME}" \ + && rm "${HADOOP_TMP_FILE}" -# Kubernetes dependencies -RUN \ -if [[ "${APT_DEPS_IMAGE}" == "airflow-apt-deps-ci" ]]; then \ - curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ - && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian stretch stable" \ - && apt-get update \ - && apt-get -y install --no-install-recommends docker-ce \ - && apt-get autoremove -yqq --purge \ - && apt-get clean && rm -rf /var/lib/apt/lists/* \ - ;\ -fi +ENV HIVE_URL="${HADOOP_URL}hive-${HIVE_VERSION}-${HADOOP_DISTRO}${HADOOP_DISTRO_VERSION}.tar.gz" \ + HIVE_TMP_FILE="/tmp/hive.tar.gz" -RUN \ -if [[ "${APT_DEPS_IMAGE}" == "airflow-apt-deps-ci" ]]; then \ - curl -Lo kubectl \ - "https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" \ - && chmod +x kubectl \ - && mv kubectl /usr/local/bin/kubectl \ - ;\ -fi +RUN curl -sL "${HIVE_URL}" >"${HIVE_TMP_FILE}" \ + && tar xzf "${HIVE_TMP_FILE}" --strip-components 1 -C "${HIVE_HOME}" \ + && rm "${HIVE_TMP_FILE}" -RUN \ -if [[ "${APT_DEPS_IMAGE}" == "airflow-apt-deps-ci" ]]; then \ - curl -Lo kind \ - "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64" \ - && chmod +x kind \ - && mv kind /usr/local/bin/kind \ - ;\ -fi - -ENV HADOOP_DISTRO=cdh \ - HADOOP_MAJOR=5 \ - HADOOP_DISTRO_VERSION=5.11.0 \ - HADOOP_VERSION=2.6.0 \ - HIVE_VERSION=1.1.0 -ENV HADOOP_URL=https://archive.cloudera.com/${HADOOP_DISTRO}${HADOOP_MAJOR}/${HADOOP_DISTRO}/${HADOOP_MAJOR}/ -ENV HADOOP_HOME=/tmp/hadoop-cdh HIVE_HOME=/tmp/hive - -RUN if [[ "${APT_DEPS_IMAGE}" == "airflow-apt-deps-ci" ]]; then /tmp/ci_build_install_deps.sh; fi +ENV MINICLUSTER_URL="${MINICLUSTER_BASE}${MINICLUSTER_VER}/minicluster-${MINICLUSTER_VER}-SNAPSHOT-bin.zip" \ + MINICLUSTER_TMP_FILE="/tmp/minicluster.zip" + +RUN curl -sL "${MINICLUSTER_URL}" > "${MINICLUSTER_TMP_FILE}" \ + && unzip "${MINICLUSTER_TMP_FILE}" -d "/tmp" \ + && rm "${MINICLUSTER_TMP_FILE}" ENV PATH "${PATH}:/tmp/hive/bin" -############################################################################################################ -# This is the target image - it installs PIP and NPM dependencies including efficient caching -# mechanisms - it might be used to build the bare airflow build or CI build -# Parameters: -# APT_DEPS_IMAGE - image with APT dependencies. It might either be base deps image with airflow -# dependencies or CI deps image that contains also CI-required dependencies -############################################################################################################ -FROM ${APT_DEPS_IMAGE} as main +RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ + && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian stretch stable" \ + && apt-get update \ + && apt-get -y install --no-install-recommends docker-ce \ + && apt-get autoremove -yqq --purge \ + && apt-get clean && rm -rf /var/lib/apt/lists/* -SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] +ARG KUBECTL_VERSION="v1.15.0" +ENV KUBECTL_VERSION=${KUBECTL_VERSION} +ARG KIND_VERSION="v0.5.0" +ENV KIND_VERSION=${KIND_VERSION} -RUN echo "Airflow version: ${AIRFLOW_VERSION}" +RUN curl -Lo kubectl \ + "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \ + && chmod +x kubectl \ + && mv kubectl /usr/local/bin/kubectl + +RUN curl -Lo kind \ + "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64" \ + && chmod +x kind \ + && mv kind /usr/local/bin/kind ARG AIRFLOW_USER=airflow ENV AIRFLOW_USER=${AIRFLOW_USER} @@ -366,15 +337,11 @@ RUN if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \ COPY --chown=airflow:airflow ./scripts/docker/entrypoint.sh /entrypoint.sh -ARG APT_DEPS_IMAGE="airflow-apt-deps-ci-slim" -ENV APT_DEPS_IMAGE=${APT_DEPS_IMAGE} - COPY --chown=airflow:airflow .bash_completion run-tests-complete run-tests ${HOME}/ COPY --chown=airflow:airflow .bash_completion.d/run-tests-complete \ ${HOME}/.bash_completion.d/run-tests-complete -RUN if [[ "${APT_DEPS_IMAGE}" == "airflow-apt-deps-ci" ]]; then \ - ${AIRFLOW_SOURCES}/scripts/ci/docker_build/ci_build_extract_tests.sh; fi +RUN ${AIRFLOW_SOURCES}/scripts/ci/docker_build/ci_build_extract_tests.sh USER ${AIRFLOW_USER} diff --git a/breeze b/breeze index d27cd5dfbafb4..442cb311698cb 100755 --- a/breeze +++ b/breeze @@ -105,10 +105,7 @@ export AIRFLOW_CONTAINER_PUSH_IMAGES=${AIRFLOW_CONTAINER_PUSH_IMAGES:="false"} # For local builds we fix file permissions only for setup-related files export AIRFLOW_FIX_PERMISSIONS=${AIRFLOW_FIX_PERMISSIONS:="setup"} -# Skip building slim image locally - we only need full CI image -export AIRFLOW_CONTAINER_SKIP_CI_SLIM_IMAGE="true" - -# Skip building full CI image locally - we only need slim image +# Skip building full CI image locally export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="false" # Branch name of the base image used (usually master or v1-10-test or v1-10-stable) @@ -682,7 +679,7 @@ echo CMDNAME="$(basename -- "$0")" # Cleans up the answer that was given last time, whether to force/ -cleanup_last_force_answer +forget_last_answer export ENV="${ENV:=$(read_from_file ENV)}" export BACKEND="${BACKEND:=$(read_from_file BACKEND)}" @@ -935,7 +932,6 @@ prepare_command_file "${BUILD_CACHE_DIR}/${LAST_DC_TEST_FILE}" "${DC_RUN_COMMAND prepare_command_file "${BUILD_CACHE_DIR}/${LAST_DC_FILE}" '"' "false" rebuild_ci_image_if_needed -rebuild_ci_slim_image_if_needed rebuild_checklicence_image_if_needed export AIRFLOW_CONTAINER_DOCKER_IMAGE=\ diff --git a/hooks/build b/hooks/build index 6f3b56fd1a630..91e0917c37d80 100755 --- a/hooks/build +++ b/hooks/build @@ -98,30 +98,18 @@ function build_python_image { NAME="${1}" MY_IMAGE_TAG="${2}" TARGET_IMAGE="${3}" - APT_DEPS_IMAGE="${4:-airflow-apt-deps-ci-slim}" - AIRFLOW_EXTRAS="${5:-all}" - AIRFLOW_USER="${6:-airflow}" - HOME="${7:-/home/airflow}" + AIRFLOW_EXTRAS="${4:-all}" + AIRFLOW_USER="${5:-airflow}" + HOME="${6:-/home/airflow}" echo "Build ${NAME} image: ${MY_IMAGE_TAG}" echo "Python base image: ${PYTHON_BASE_IMAGE}" set +u - if [[ "${MY_IMAGE_TAG}" == "${AIRFLOW_CI_IMAGE}" ]]; then - DOCKER_CACHE_DIRECTIVE=("${DOCKER_CACHE_DIRECTIVE_CI[@]}") - elif [[ "${MY_IMAGE_TAG}" == "${AIRFLOW_SLIM_CI_IMAGE}" ]]; then - DOCKER_CACHE_DIRECTIVE=("${DOCKER_CACHE_DIRECTIVE_CI_SLIM[@]}") - else - echo - echo "Don't know how to set cache directive for ${MY_IMAGE_TAG}. Exiting" - echo - exit 1 - fi set -x docker build \ --build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE}" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ - --build-arg APT_DEPS_IMAGE="${APT_DEPS_IMAGE}" \ --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ --build-arg AIRFLOW_USER="${AIRFLOW_USER}" \ --build-arg AIRFLOW_BRANCH="${AIRFLOW_CONTAINER_BRANCH_NAME}" \ @@ -189,9 +177,6 @@ else NON_CI="true" fi -# Extras used to to build slim airflow image -AIRFLOW_SLIM_EXTRAS=${AIRFLOW_SLIM_EXTRAS:="all"} - # Extras used to build cache and CI image AIRFLOW_CI_EXTRAS=${AIRFLOW_CI_EXTRAS:="devel_ci"} @@ -212,13 +197,8 @@ export AIRFLOW_CONTAINER_FORCE_PULL_IMAGES=${AIRFLOW_CONTAINER_FORCE_PULL_IMAGES # In CI environment (and local force) we skip pulling latest python image export AIRFLOW_CONTAINER_SKIP_LATEST_PYTHON_PULL=${AIRFLOW_CONTAINER_SKIP_LATEST_PYTHON_PULL:=${CI}} -# Skips downloading and building the slim image of airflow -# This is set to true by default in CI test environment (we only need full CI image then) -export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE=${AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE:=${CI}} -echo "Skip slim image: ${AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE}" - # Skips downloading and building the CI image, full CI-enabled image of airflow -# This is set to true for pre-tests which do not need the whole CI image just the slim one +# This is set to true for pre-tests which do not need the whole CI image export AIRFLOW_CONTAINER_SKIP_CI_IMAGE=${AIRFLOW_CONTAINER_SKIP_CI_IMAGE:="false"} echo "Skip CI image: ${AIRFLOW_CONTAINER_SKIP_CI_IMAGE}" @@ -241,11 +221,6 @@ export PYTHON_BASE_IMAGE=python:${PYTHON_VERSION}-slim-stretch export CHECKLICENCE_BASE_IMAGE=debian:stretch-slim if [[ "${AIRFLOW_RELEASE_BUILD}" == "true" ]]; then - export AIRFLOW_SLIM_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_VERSION}-python${PYTHON_VERSION}-ci-slim" - export AIRFLOW_SLIM_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_VERSION}-ci-slim" - export AIRFLOW_SLIM_CI_IMAGE_LATEST="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:latest-python${PYTHON_VERSION}-${TAG_PREFIX}-ci-slim" - export AIRFLOW_SLIM_CI_IMAGE_LATEST_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:latest-${TAG_PREFIX}-ci-slim" - export AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_VERSION}-python${PYTHON_VERSION}-ci" export AIRFLOW_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_VERSION}-ci" export AIRFLOW_CI_IMAGE_LATEST="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:latest-python${PYTHON_VERSION}-${TAG_PREFIX}-ci" @@ -253,9 +228,6 @@ if [[ "${AIRFLOW_RELEASE_BUILD}" == "true" ]]; then export AIRFLOW_CHECKLICENCE_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:checklicence" else - export AIRFLOW_SLIM_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${TAG_PREFIX}-python${PYTHON_VERSION}-ci-slim" - export AIRFLOW_SLIM_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${TAG_PREFIX}-ci-slim" - export AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${TAG_PREFIX}-python${PYTHON_VERSION}-ci" export AIRFLOW_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${TAG_PREFIX}-ci" export AIRFLOW_CHECKLICENCE_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:checklicence" @@ -280,7 +252,6 @@ if [[ "${AIRFLOW_CONTAINER_CLEANUP_IMAGES}" == "true" ]]; then start_step "Removing images" docker rmi "${PYTHON_BASE_IMAGE}" || true docker rmi "${CHECKLICENCE_BASE_IMAGE}" || true - docker rmi "${AIRFLOW_SLIM_CI_IMAGE}" || true docker rmi "${AIRFLOW_CI_IMAGE}" || true docker rmi "${AIRFLOW_CHECKLICENCE_IMAGE}" || true echo @@ -297,7 +268,6 @@ fi start_step "Populating cache" DOCKER_CACHE_DIRECTIVE_CI=() -DOCKER_CACHE_DIRECTIVE_CI_SLIM=() DOCKER_CACHE_DIRECTIVE_CHECKLICENCE=() if [[ "${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE}" == "true" ]]; then @@ -328,17 +298,12 @@ if [[ "${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE}" == "true" ]]; then echo "Downloading the CI Airflow image" IMAGES_TO_PULL="${IMAGES_TO_PULL} ${AIRFLOW_CI_IMAGE}" fi - if [[ "${AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE}" != "true" ]]; then - echo "Downloading the slim Airflow image" - IMAGES_TO_PULL="${IMAGES_TO_PULL} ${AIRFLOW_SLIM_CI_IMAGE}" - fi if [[ "${AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE}" != "true" ]]; then echo "Downloading the CHECKLICENCE Airflow image" IMAGES_TO_PULL="${IMAGES_TO_PULL} ${AIRFLOW_CHECKLICENCE_IMAGE}" fi DOCKER_CACHE_DIRECTIVE_CI=() - DOCKER_CACHE_DIRECTIVE_CI_SLIM=() if [[ ${IMAGES_TO_PULL} == "" ]]; then echo echo "Skipping building of all images." @@ -378,8 +343,6 @@ if [[ "${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE}" == "true" ]]; then fi if [[ "${IMAGE}" == "${AIRFLOW_CI_IMAGE}" ]]; then DOCKER_CACHE_DIRECTIVE_CI+=("--cache-from" "${IMAGE}") - elif [[ "${IMAGE}" == "${AIRFLOW_SLIM_CI_IMAGE}" ]]; then - DOCKER_CACHE_DIRECTIVE_CI_SLIM+=("--cache-from" "${IMAGE}") elif [[ "${IMAGE}" == "${AIRFLOW_CHECKLICENCE_IMAGE}" ]]; then DOCKER_CACHE_DIRECTIVE_CHECKLICENCE+=("--cache-from" "${IMAGE}") else @@ -396,14 +359,12 @@ start_step "Setting cache options" if [[ "${AIRFLOW_CONTAINER_USE_NO_CACHE}" == "true" ]]; then DOCKER_CACHE_DIRECTIVE_CI+=("--no-cache") - DOCKER_CACHE_DIRECTIVE_CI_SLIM+=("--no-cache") DOCKER_CACHE_DIRECTIVE_CHECKLICENCE+=("--no-cache") echo echo "Skip cache for builds. Everything will be rebuilt from scratch." echo echo "Cache directives used: " echo "CI build: ${DOCKER_CACHE_DIRECTIVE_CI[*]}" - echo "CI slim build: ${DOCKER_CACHE_DIRECTIVE_CI_SLIM[*]}" echo "CI checklicence build: ${DOCKER_CACHE_DIRECTIVE_CHECKLICENCE[*]}" echo elif [[ "${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE}" == "true" ]]; then @@ -412,13 +373,11 @@ elif [[ "${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE}" == "true" ]]; then echo "Cache directives used: " set +u echo "CI build: ${DOCKER_CACHE_DIRECTIVE_CI[*]}" - echo "CI slim build: ${DOCKER_CACHE_DIRECTIVE_CI_SLIM[*]}" echo "CI checklicence build: ${DOCKER_CACHE_DIRECTIVE_CHECKLICENCE[*]}" set -u echo else DOCKER_CACHE_DIRECTIVE_CI=() - DOCKER_CACHE_DIRECTIVE_CI_SLIM=() DOCKER_CACHE_DIRECTIVE_CHECKLICENCE=() echo echo "Use default cache from locally built images." @@ -426,7 +385,6 @@ else echo "Cache directives used: " set +u echo "CI build: ${DOCKER_CACHE_DIRECTIVE_CI[*]}" - echo "CI slim build: ${DOCKER_CACHE_DIRECTIVE_CI_SLIM[*]}" echo "CI checklicence build: ${DOCKER_CACHE_DIRECTIVE_CHECKLICENCE[*]}" set -u echo @@ -443,7 +401,7 @@ BUILD_ID=${BUILD_ID:="local"} # directory where "deployment" artifacts should be placed -DEPLOY_DIR=${AIRFLOW_SOURCES}/dist/${AIRFLOW_CONTAINER_BRANCH_NAME}/$(date +%Y-%m-%d)/${BUILD_ID}/${PYTHON_VERSION} +DEPLOY_DIR="${AIRFLOW_SOURCES}/dist/${AIRFLOW_CONTAINER_BRANCH_NAME}/$(date "+%Y-%m-%d")/${BUILD_ID}/${PYTHON_VERSION}" mkdir -pv "${DEPLOY_DIR}" @@ -483,29 +441,6 @@ echo "Group/other write access removed for ${FILES_FOR_REBUILD_CHECK[*]}" echo echo -start_step "Build Airflow CI slim image" - -if [[ "${AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE}" == "true" ]]; then - echo "Skip building the Airflow CI slim image" -else - build_python_image "Airflow" \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ - "main" \ - "airflow-apt-deps-ci-slim" - if [[ "${PYTHON_VERSION_FOR_DEFAULT_IMAGE}" == "${PYTHON_VERSION}" ]]; then - docker tag "${AIRFLOW_SLIM_CI_IMAGE}" "${AIRFLOW_SLIM_CI_IMAGE_DEFAULT}" - add_image_to_push "${AIRFLOW_SLIM_CI_IMAGE_DEFAULT}" - fi - if [[ "${AIRFLOW_RELEASE_BUILD}" == "true" ]]; then - docker tag "${AIRFLOW_SLIM_CI_IMAGE}" "${AIRFLOW_SLIM_CI_IMAGE_LATEST}" - add_image_to_push "${AIRFLOW_SLIM_CI_IMAGE_LATEST}" - if [[ "${PYTHON_VERSION_FOR_DEFAULT_IMAGE}" == "${PYTHON_VERSION}" ]]; then - docker tag "${AIRFLOW_SLIM_CI_IMAGE}" "${AIRFLOW_SLIM_CI_IMAGE_LATEST_DEFAULT}" - add_image_to_push "${AIRFLOW_SLIM_CI_IMAGE_LATEST_DEFAULT}" - fi - fi -fi - if [[ "${AIRFLOW_CONTAINER_SKIP_CI_IMAGE}" == "true" ]]; then echo "Skip building the CI full Airflow image" else @@ -513,7 +448,6 @@ else build_python_image "Airflow CI" \ "${AIRFLOW_CI_IMAGE}" \ "main" \ - "airflow-apt-deps-ci" \ "devel_ci" \ "root" \ "/root" @@ -536,8 +470,10 @@ if [[ "${AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE}" == "true" ]]; then echo "Skip building the CHECKLICENCE image" else start_step "Build Airflow CHECKLICENCE image" + set +u docker build . -f Dockerfile-checklicence \ "${DOCKER_CACHE_DIRECTIVE_CHECKLICENCE[@]}" -t "${AIRFLOW_CHECKLICENCE_IMAGE}" + set -u add_image_to_push "${AIRFLOW_CHECKLICENCE_IMAGE}" fi diff --git a/scripts/ci/_utils.sh b/scripts/ci/_utils.sh index 3a12c0c3b4cd7..0daa85027654c 100644 --- a/scripts/ci/_utils.sh +++ b/scripts/ci/_utils.sh @@ -33,7 +33,7 @@ export BUILD_CACHE_DIR LAST_FORCE_ANSWER_FILE="${BUILD_CACHE_DIR}/last_force_answer.sh" -IMAGES_TO_CHECK=("SLIM_CI" "CI" "CHECKLICENCE") +IMAGES_TO_CHECK=("CI" "CHECKLICENCE") export IMAGES_TO_CHECK mkdir -p "${AIRFLOW_SOURCES}/.mypy_cache" @@ -236,11 +236,8 @@ function update_all_md5_files() { # or the Dockerfile itself changes. # # Another reason to skip rebuilding Docker is thar currently it takes a bit longer time than simple Docker -# files. There are the following, problems with the current Dockerfiles that need longer build times: -# 1) We need to fix group permissions of files in Docker because different linux build services have -# different default umask and Docker uses group permissions in checking for cache invalidation. -# 2) we use multi-stage build and in case of slim image we needlessly build a full CI image because -# support for this only comes with the upcoming buildkit: https://github.com/docker/cli/issues/1134 +# We need to fix group permissions of files in Docker because different linux build services have +# different default umask and Docker uses group permissions in checking for cache invalidation. # # As result of this check - most of the static checks will start pretty much immediately. # @@ -355,29 +352,29 @@ function assert_not_in_container() { fi } -function cleanup_last_force_answer() { - # Removes the "Force answer" (yes/no/quit) given previously, unles you specifically want to use it. +function forget_last_answer() { + # Removes the "Forced answer" (yes/no/quit) given previously, unles you specifically want to remember it. # # This is the default behaviour of all rebuild scripts to ask independently whether you want to - # rebuild the image or not. Sometimes however we want to reuse answer previously given. For + # rebuild the image or not. Sometimes however we want to remember answer previously given. For # example if you answered "no" to rebuild the image, the assumption is that you do not - # want to rebuild image for other rebuilds in the same pre-commit execution. + # want to rebuild image also for other rebuilds in the same pre-commit execution. # - # All the pre-commit checks therefore have `export SKIP_CLEANUP_OF_LAST_ANSWER="true"` set + # All the pre-commit checks therefore have `export REMEMBER_LAST_ANSWER="true"` set # So that in case they are run in a sequence of commits they will not rebuild. Similarly if your most # recent answer was "no" and you run `pre-commit run mypy` (for example) it will also reuse the # "no" answer given previously. This happens until you run any of the breeze commands or run all # precommits `pre-commit run` - then the "LAST_FORCE_ANSWER_FILE" will be removed and you will # be asked again. - if [[ ${SKIP_CLEANUP_OF_LAST_ANSWER:=""} != "true" ]]; then + if [[ ${REMEMBER_LAST_ANSWER:="false"} != "true" ]]; then print_info - print_info "Removing last answer from ${LAST_FORCE_ANSWER_FILE}" + print_info "Forgetting last answer from ${LAST_FORCE_ANSWER_FILE}:" print_info rm -f "${LAST_FORCE_ANSWER_FILE}" else if [[ -f "${LAST_FORCE_ANSWER_FILE}" ]]; then print_info - print_info "Retaining last answer from ${LAST_FORCE_ANSWER_FILE}" + print_info "Still remember last answer from ${LAST_FORCE_ANSWER_FILE}:" print_info "$(cat "${LAST_FORCE_ANSWER_FILE}")" print_info fi @@ -493,47 +490,10 @@ EOF fi } -# -# Rebuilds the slim image for static checks if needed. In order to speed it up, it's built without NPM -# -function rebuild_ci_slim_image_if_needed() { - export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE="false" - export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="true" - export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE="true" - - # Temporary force python version 3.5 for static checks - export OLD_PYTHON_VERSION=${PYTHON_VERSION=""} - export PYTHON_VERSION=3.5 - - export THE_IMAGE_TYPE="SLIM_CI" - - rebuild_image_if_needed - - export AIRFLOW_SLIM_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${DEFAULT_BRANCH}-python${PYTHON_VERSION}-ci-slim" - export PYTHON_VERSION=${OLD_PYTHON_VERSION} -} - -# -# Cleans up the CI slim image -# -function cleanup_ci_slim_image() { - export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE="false" - export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="true" - export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE="true" - export AIRFLOW_CONTAINER_CLEANUP_IMAGES="true" - - export PYTHON_VERSION=3.5 # Always use python version 3.5 for static checks - - export THE_IMAGE_TYPE="SLIM_CI" - - rebuild_image_if_needed -} - # # Rebuilds the image for tests if needed. # function rebuild_ci_image_if_needed() { - export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE="true" export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE="true" export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="false" @@ -546,10 +506,9 @@ function rebuild_ci_image_if_needed() { # -# Cleans up the CI slim image +# Cleans up the CI image # function cleanup_ci_image() { - export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE="true" export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="false" export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE="true" export AIRFLOW_CONTAINER_CLEANUP_IMAGES="true" @@ -563,7 +522,6 @@ function cleanup_ci_image() { # Rebuilds the image for licence checks if needed. # function rebuild_checklicence_image_if_needed() { - export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE="true" export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE="false" export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="true" @@ -575,10 +533,9 @@ function rebuild_checklicence_image_if_needed() { } # -# Cleans up the CI slim image +# Cleans up the checklicence image # function cleanup_checklicence_image() { - export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE="true" export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="true" export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE="false" export AIRFLOW_CONTAINER_CLEANUP_IMAGES="true" @@ -648,7 +605,7 @@ function basic_sanity_checks() { go_to_airflow_sources check_if_coreutils_installed create_cache_directory - cleanup_last_force_answer + forget_last_answer } @@ -664,7 +621,7 @@ function run_flake8() { --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ + "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" \ | tee -a "${OUTPUT_LOG}" else @@ -676,7 +633,7 @@ function run_flake8() { --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ + "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" "${FILES[@]}" \ | tee -a "${OUTPUT_LOG}" fi @@ -691,7 +648,7 @@ function run_docs() { --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ + "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/docs/build.sh" \ | tee -a "${OUTPUT_LOG}" } @@ -721,7 +678,7 @@ function run_mypy() { --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ + "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/ci/in_container/run_mypy.sh" "airflow" "tests" "docs" \ | tee -a "${OUTPUT_LOG}" else @@ -733,7 +690,7 @@ function run_mypy() { --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ + "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/ci/in_container/run_mypy.sh" "${FILES[@]}" \ | tee -a "${OUTPUT_LOG}" fi @@ -750,7 +707,7 @@ function run_pylint_main() { --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ + "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" \ | tee -a "${OUTPUT_LOG}" else @@ -762,7 +719,7 @@ function run_pylint_main() { --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ + "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" "${FILES[@]}" \ | tee -a "${OUTPUT_LOG}" fi @@ -780,7 +737,7 @@ function run_pylint_tests() { --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ + "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" \ | tee -a "${OUTPUT_LOG}" else @@ -792,7 +749,7 @@ function run_pylint_tests() { --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ + "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" "${FILES[@]}" \ | tee -a "${OUTPUT_LOG}" fi @@ -852,7 +809,7 @@ function refresh_pylint_todo() { --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ - "${AIRFLOW_SLIM_CI_IMAGE}" | tee -a "${OUTPUT_LOG}" + "${AIRFLOW_CI_IMAGE}" | tee -a "${OUTPUT_LOG}" } function rebuild_all_images_if_needed_and_confirmed() { @@ -861,15 +818,7 @@ function rebuild_all_images_if_needed_and_confirmed() { for THE_IMAGE_TYPE in "${IMAGES_TO_CHECK[@]}" do - if [[ "${THE_IMAGE_TYPE}" == "SLIM_CI" ]]; then - # Temporary force python version 3.5 for static checks - export OLD_PYTHON_VERSION=${PYTHON_VERSION=""} - export PYTHON_VERSION=3.5 - - check_if_docker_build_is_needed - - export PYTHON_VERSION=${OLD_PYTHON_VERSION} - fi + check_if_docker_build_is_needed done if [[ ${AIRFLOW_CONTAINER_DOCKER_BUILD_NEEDED} == "true" ]]; then @@ -901,7 +850,6 @@ function rebuild_all_images_if_needed_and_confirmed() { if [[ ${SKIP_REBUILD} != "true" ]]; then rebuild_ci_image_if_needed - rebuild_ci_slim_image_if_needed rebuild_checklicence_image_if_needed fi fi @@ -969,16 +917,16 @@ function build_image_on_ci() { elif [[ ${TRAVIS_JOB_NAME} == "Check lic"* ]]; then rebuild_checklicence_image_if_needed elif [[ ${TRAVIS_JOB_NAME} == "Static"* ]]; then - rebuild_ci_slim_image_if_needed + rebuild_ci_image_if_needed elif [[ ${TRAVIS_JOB_NAME} == "Pylint"* ]]; then match_files_regexp '.*\.py' if [[ ${FILE_MATCHES} == "true" || ${TRAVIS_PULL_REQUEST:=} == "false" ]]; then - rebuild_ci_slim_image_if_needed + rebuild_ci_image_if_needed else touch "${BUILD_CACHE_DIR}"/.skip_tests fi elif [[ ${TRAVIS_JOB_NAME} == "Build documentation"* ]]; then - rebuild_ci_slim_image_if_needed + rebuild_ci_image_if_needed else echo echo "Error! Unexpected Travis job name: ${TRAVIS_JOB_NAME}" diff --git a/scripts/ci/ci_docs.sh b/scripts/ci/ci_docs.sh index 603f19b86811c..bc6c48cffd7cd 100755 --- a/scripts/ci/ci_docs.sh +++ b/scripts/ci/ci_docs.sh @@ -31,7 +31,7 @@ basic_sanity_checks script_start -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed run_docs diff --git a/scripts/ci/ci_flake8.sh b/scripts/ci/ci_flake8.sh index 958305171b9f8..553043090db61 100755 --- a/scripts/ci/ci_flake8.sh +++ b/scripts/ci/ci_flake8.sh @@ -31,7 +31,7 @@ basic_sanity_checks script_start -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed run_flake8 "$@" diff --git a/scripts/ci/ci_isort.sh b/scripts/ci/ci_isort.sh index a8ada59956858..01e7b1163da31 100755 --- a/scripts/ci/ci_isort.sh +++ b/scripts/ci/ci_isort.sh @@ -32,7 +32,7 @@ force_python_3_5 script_start -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed run_flake8 "$@" diff --git a/scripts/ci/ci_mypy.sh b/scripts/ci/ci_mypy.sh index 6f0837c83a82b..8d4dd22b4cb17 100755 --- a/scripts/ci/ci_mypy.sh +++ b/scripts/ci/ci_mypy.sh @@ -31,7 +31,7 @@ basic_sanity_checks script_start -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed run_mypy "$@" diff --git a/scripts/ci/ci_pylint_main.sh b/scripts/ci/ci_pylint_main.sh index 8c2d533047801..430effd0d2b83 100755 --- a/scripts/ci/ci_pylint_main.sh +++ b/scripts/ci/ci_pylint_main.sh @@ -31,7 +31,7 @@ basic_sanity_checks script_start -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed if [[ "${#@}" != "0" ]]; then filter_out_files_from_pylint_todo_list "$@" diff --git a/scripts/ci/ci_pylint_tests.sh b/scripts/ci/ci_pylint_tests.sh index 2edc53f5ebd26..df7c189929700 100755 --- a/scripts/ci/ci_pylint_tests.sh +++ b/scripts/ci/ci_pylint_tests.sh @@ -31,7 +31,7 @@ basic_sanity_checks script_start -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed if [[ "${#@}" != "0" ]]; then filter_out_files_from_pylint_todo_list "$@" diff --git a/scripts/ci/ci_refresh_pylint_todo.sh b/scripts/ci/ci_refresh_pylint_todo.sh index bbeed2c843ab6..46716c3150884 100755 --- a/scripts/ci/ci_refresh_pylint_todo.sh +++ b/scripts/ci/ci_refresh_pylint_todo.sh @@ -30,7 +30,7 @@ basic_sanity_checks script_start -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed refresh_pylint_todo diff --git a/scripts/ci/ci_run_all_static_tests.sh b/scripts/ci/ci_run_all_static_tests.sh index 6e80267556d34..03ae9aca5057c 100755 --- a/scripts/ci/ci_run_all_static_tests.sh +++ b/scripts/ci/ci_run_all_static_tests.sh @@ -39,10 +39,10 @@ if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then exit fi -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed rebuild_checklicence_image_if_needed -IMAGES_TO_CHECK=("SLIM_CI" "CHECKLICENCE") +IMAGES_TO_CHECK=("CI" "CHECKLICENCE") export IMAGES_TO_CHECK pre-commit run --all-files --show-diff-on-failure diff --git a/scripts/ci/ci_run_all_static_tests_except_pylint.sh b/scripts/ci/ci_run_all_static_tests_except_pylint.sh index ec26b0679ba4f..e6943b79b2111 100755 --- a/scripts/ci/ci_run_all_static_tests_except_pylint.sh +++ b/scripts/ci/ci_run_all_static_tests_except_pylint.sh @@ -39,10 +39,10 @@ if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then exit fi -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed rebuild_checklicence_image_if_needed -IMAGES_TO_CHECK=("SLIM_CI" "CHECKLICENCE") +IMAGES_TO_CHECK=("CI" "CHECKLICENCE") export IMAGES_TO_CHECK SKIP=pylint pre-commit run --all-files --show-diff-on-failure diff --git a/scripts/ci/ci_run_all_static_tests_except_pylint_licence.sh b/scripts/ci/ci_run_all_static_tests_except_pylint_licence.sh index 55e71b1ac60aa..0899eb6210222 100755 --- a/scripts/ci/ci_run_all_static_tests_except_pylint_licence.sh +++ b/scripts/ci/ci_run_all_static_tests_except_pylint_licence.sh @@ -39,9 +39,9 @@ if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then exit fi -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed -IMAGES_TO_CHECK=("SLIM_CI") +IMAGES_TO_CHECK=("CI") export IMAGES_TO_CHECK SKIP=pylint,check-apache-license pre-commit run --all-files --show-diff-on-failure diff --git a/scripts/ci/ci_run_all_static_tests_pylint.sh b/scripts/ci/ci_run_all_static_tests_pylint.sh index 2b9fd03fb6667..5bb8a655e552e 100755 --- a/scripts/ci/ci_run_all_static_tests_pylint.sh +++ b/scripts/ci/ci_run_all_static_tests_pylint.sh @@ -39,9 +39,9 @@ if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then exit fi -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed -IMAGES_TO_CHECK=("SLIM_CI") +IMAGES_TO_CHECK=("CI") export IMAGES_TO_CHECK pre-commit run pylint --all-files --show-diff-on-failure diff --git a/scripts/ci/docker_build/ci_build_install_deps.sh b/scripts/ci/docker_build/ci_build_install_deps.sh deleted file mode 100755 index eb690822872e3..0000000000000 --- a/scripts/ci/docker_build/ci_build_install_deps.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Bash sanity settings (error on exit, complain for undefined vars, error when pipe fails) -set -euxo pipefail - -# TODO: We should think about removing those and moving them into docker-compose dependencies. -# TODO: We might come up with just one airflow CI image not the SLIM/CI versions. That would simplify a lot. -# TODO: We could likely get rid of the multi-staging approach. It introduces a number of limitations -# TODO: As long as we decrease the size of the CI image, we should be fine with using single CI image for -# TODO: everything. Currently the CI image is about 1GB where CI_SLIM is around 0.5 GB. - -export HADOOP_DISTRO="${HADOOP_DISTRO:="cdh"}" -export HADOOP_MAJOR="${HADOOP_MAJOR:="5"}" -export HADOOP_DISTRO_VERSION="${HADOOP_DISTRO_VERSION:="5.11.0"}" -export HADOOP_VERSION="${HADOOP_VERSION:="2.6.0"}" -export HIVE_VERSION="${HIVE_VERSION:="1.1.0"}" -export HADOOP_URL="${HADOOP_URL:="https://archive.cloudera.com/${HADOOP_DISTRO}${HADOOP_MAJOR}/${HADOOP_DISTRO}/${HADOOP_MAJOR}/"}" -export HADOOP_HOME="${HADOOP_HOME:="/tmp/hadoop-cdh"}" -export HIVE_HOME="${HIVE_HOME:="/tmp/hive"}" -export MINICLUSTER_BASE="${MINICLUSTER_BASE:="https://github.com/bolkedebruin/minicluster/releases/download/"}" -export MINICLUSTER_HOME="${MINICLUSTER_HOME:="/tmp/minicluster"}" -export MINICLUSTER_VER="${MINICLUSTER_VER:="1.1"}" - -mkdir -pv "${HADOOP_HOME}" -mkdir -pv "${HIVE_HOME}" -mkdir -pv "${MINICLUSTER_HOME}" -mkdir -pv "/user/hive/warehouse" -chmod -R 777 "${HIVE_HOME}" -chmod -R 777 "/user/" - -export HADOOP_DOWNLOAD_URL="${HADOOP_URL}hadoop-${HADOOP_VERSION}-${HADOOP_DISTRO}${HADOOP_DISTRO_VERSION}.tar.gz" -export HADOOP_TMP_FILE="/tmp/hadoop.tar.gz" - -curl -sL "${HADOOP_DOWNLOAD_URL}" >"${HADOOP_TMP_FILE}" - -tar xzf "${HADOOP_TMP_FILE}" --absolute-names --strip-components 1 -C "${HADOOP_HOME}" - -rm "${HADOOP_TMP_FILE}" - -export HIVE_URL="${HADOOP_URL}hive-${HIVE_VERSION}-${HADOOP_DISTRO}${HADOOP_DISTRO_VERSION}.tar.gz" -export HIVE_TMP_FILE="/tmp/hive.tar.gz" - -curl -sL "${HIVE_URL}" >"${HIVE_TMP_FILE}" -tar xzf "${HIVE_TMP_FILE}" --strip-components 1 -C "${HIVE_HOME}" -rm "${HIVE_TMP_FILE}" - -MINICLUSTER_URL="${MINICLUSTER_BASE}${MINICLUSTER_VER}/minicluster-${MINICLUSTER_VER}-SNAPSHOT-bin.zip" -MINICLUSTER_TMP_FILE="/tmp/minicluster.zip" - -curl -sL "${MINICLUSTER_URL}" > "${MINICLUSTER_TMP_FILE}" -unzip "${MINICLUSTER_TMP_FILE}" -d "/tmp" -rm "${MINICLUSTER_TMP_FILE}" diff --git a/scripts/ci/in_container/run_pylint_main.sh b/scripts/ci/in_container/run_pylint_main.sh index f8935c973e704..198636ca8d1f1 100755 --- a/scripts/ci/in_container/run_pylint_main.sh +++ b/scripts/ci/in_container/run_pylint_main.sh @@ -41,6 +41,7 @@ if [[ ${#@} == "0" ]]; then -path "./airflow/www/node_modules" -prune -o \ -path "./airflow/www_rbac/node_modules" -prune -o \ -path "./airflow/_vendor" -prune -o \ + -path "./airflow/migrations/versions" -prune -o \ -path "./.eggs" -prune -o \ -path "./docs/_build" -prune -o \ -path "./build" -prune -o \ diff --git a/scripts/ci/local_ci_build_ci_slim_image.sh b/scripts/ci/local_ci_build_ci_slim_image.sh index 6c0635551b42a..b632f0009e2cf 100755 --- a/scripts/ci/local_ci_build_ci_slim_image.sh +++ b/scripts/ci/local_ci_build_ci_slim_image.sh @@ -30,6 +30,6 @@ basic_sanity_checks script_start -rebuild_ci_slim_image_if_needed +rebuild_ci_image_if_needed script_end diff --git a/scripts/ci/local_ci_cleanup.sh b/scripts/ci/local_ci_cleanup.sh index 388731b9619bd..9600c5d3b5609 100755 --- a/scripts/ci/local_ci_cleanup.sh +++ b/scripts/ci/local_ci_cleanup.sh @@ -32,8 +32,6 @@ script_start cleanup_ci_image -cleanup_ci_slim_image - cleanup_checklicence_image script_end diff --git a/scripts/ci/pre_commit_check_license.sh b/scripts/ci/pre_commit_check_license.sh index da38ea7fb50bd..fbcff1497cb7e 100755 --- a/scripts/ci/pre_commit_check_license.sh +++ b/scripts/ci/pre_commit_check_license.sh @@ -20,6 +20,6 @@ set -uo pipefail MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"} -export SKIP_CLEANUP_OF_LAST_ANSWER="true" +export REMEMBER_LAST_ANSWER="true" "${MY_DIR}/ci_check_license.sh" "${@}" diff --git a/scripts/ci/pre_commit_ci_build.sh b/scripts/ci/pre_commit_ci_build.sh new file mode 100755 index 0000000000000..217434ba77240 --- /dev/null +++ b/scripts/ci/pre_commit_ci_build.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -uo pipefail + +MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +export REMEMBER_LAST_ANSWER="false" +export PYTHON_VERSION=3.5 + +"${MY_DIR}/local_ci_build.sh" diff --git a/scripts/ci/pre_commit_flake8.sh b/scripts/ci/pre_commit_flake8.sh index c8fc2f6c6d1af..5108751d5d39e 100755 --- a/scripts/ci/pre_commit_flake8.sh +++ b/scripts/ci/pre_commit_flake8.sh @@ -20,6 +20,6 @@ set -uo pipefail MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"} -export SKIP_CLEANUP_OF_LAST_ANSWER="true" +export REMEMBER_LAST_ANSWER="true" "${MY_DIR}/ci_flake8.sh" "${@}" diff --git a/scripts/ci/pre_commit_lint_dockerfile.sh b/scripts/ci/pre_commit_lint_dockerfile.sh index e742a6d672471..bb729c114f6f6 100755 --- a/scripts/ci/pre_commit_lint_dockerfile.sh +++ b/scripts/ci/pre_commit_lint_dockerfile.sh @@ -19,6 +19,6 @@ set -uo pipefail MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -export SKIP_CLEANUP_OF_LAST_ANSWER="true" +export REMEMBER_LAST_ANSWER="true" "${MY_DIR}/ci_lint_dockerfile.sh" "${@}" diff --git a/scripts/ci/pre_commit_mypy.sh b/scripts/ci/pre_commit_mypy.sh index 116a410ebec2f..d137d20fce9c0 100755 --- a/scripts/ci/pre_commit_mypy.sh +++ b/scripts/ci/pre_commit_mypy.sh @@ -20,6 +20,6 @@ set -uo pipefail MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"} -export SKIP_CLEANUP_OF_LAST_ANSWER="true" +export REMEMBER_LAST_ANSWER="true" "${MY_DIR}/ci_mypy.sh" "${@}" diff --git a/scripts/ci/pre_commit_pylint_main.sh b/scripts/ci/pre_commit_pylint_main.sh index 58b48e1c7af27..70f496fe91fbc 100755 --- a/scripts/ci/pre_commit_pylint_main.sh +++ b/scripts/ci/pre_commit_pylint_main.sh @@ -20,6 +20,6 @@ set -uo pipefail MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"} -export SKIP_CLEANUP_OF_LAST_ANSWER="true" +export REMEMBER_LAST_ANSWER="true" "${MY_DIR}/ci_pylint_main.sh" "${@}" diff --git a/scripts/ci/pre_commit_pylint_tests.sh b/scripts/ci/pre_commit_pylint_tests.sh index 7ff27ac65bd1d..dcb7386118e60 100755 --- a/scripts/ci/pre_commit_pylint_tests.sh +++ b/scripts/ci/pre_commit_pylint_tests.sh @@ -20,6 +20,6 @@ set -uo pipefail MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"} -export SKIP_CLEANUP_OF_LAST_ANSWER="true" +export REMEMBER_LAST_ANSWER="true" "${MY_DIR}/ci_pylint_tests.sh" "${@}" diff --git a/tests/test_impersonation.py b/tests/test_impersonation.py index cc723d9bd480d..c3013e9436ed9 100644 --- a/tests/test_impersonation.py +++ b/tests/test_impersonation.py @@ -80,11 +80,11 @@ def revoke_permissions(): def check_original_docker_image(): - if not os.path.isfile('/.dockerenv') or os.environ.get('APT_DEPS_IMAGE') is None: + if not os.path.isfile('/.dockerenv') or os.environ.get('PYTHON_BASE_IMAGE') is None: raise unittest.SkipTest("""Adding/removing a user as part of a test is very bad for host os (especially if the user already existed to begin with on the OS), therefore we check if we run inside a the official docker container and only allow to run the test there. This is done by checking /.dockerenv -file (always present inside container) and checking for APT_DEPS_IMAGE variable. +file (always present inside container) and checking for PYTHON_BASE_IMAGE variable. """)