Conditional MySQL Client installation (apache#11174)

This is the second step of making the Production Docker Image more corporate-environment friendly, by making MySQL client installation optional. Instaling MySQL Client on Debian requires to reach out to oracle deb repositories which might not be approved by security teams when you build the images. Also not everyone needs MySQL client or might want to install their own MySQL client or MariaDB client - from their own repositories. This change makes the installation step separated out to script (with prod/dev installation option). The prod/dev separation is needed because MySQL needs to be installed with dev libraries in the "Build" segment of the image (requiring build essentials etc.) but in "Final" segment of the image only runtime libraries are needed. Part of apache#11171 Depends on apache#11173.
yogesh1978 · Sep 27, 2020 · 044b441 · 044b441
1 parent 0db7a30
commit 044b441
Show file tree

Hide file tree

Showing 10 changed files with 113 additions and 76 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -37,6 +37,7 @@
 
 # Add those folders to the context so that they are available in the CI container
 !scripts/in_container
+!scripts/docker
 
 # Add backport packages to the context
 !backport_packages

diff --git a/BREEZE.rst b/BREEZE.rst
@@ -1241,6 +1241,10 @@ This is the current syntax for  `./breeze <./breeze>`_:
   --disable-pip-cache
           Disables GitHub PIP cache during the build. Useful if github is not reachable during build.
 
+  --disable-mysql-client-installation
+          Disables installation of the mysql client which might be problematic if you are building
+          image in controlled environment. Only valid for production image.
+
   -C, --force-clean-images
           Force build images with cache disabled. This will remove the pulled or build images
           and start building images from scratch. This might take a long time.
@@ -1706,6 +1710,10 @@ This is the current syntax for  `./breeze <./breeze>`_:
   --disable-pip-cache
           Disables GitHub PIP cache during the build. Useful if github is not reachable during build.
 
+  --disable-mysql-client-installation
+          Disables installation of the mysql client which might be problematic if you are building
+          image in controlled environment. Only valid for production image.
+
   -C, --force-clean-images
           Force build images with cache disabled. This will remove the pulled or build images
           and start building images from scratch. This might take a long time.
@@ -2113,6 +2121,10 @@ This is the current syntax for  `./breeze <./breeze>`_:
   --disable-pip-cache
           Disables GitHub PIP cache during the build. Useful if github is not reachable during build.
 
+  --disable-mysql-client-installation
+          Disables installation of the mysql client which might be problematic if you are building
+          image in controlled environment. Only valid for production image.
+
   -C, --force-clean-images
           Force build images with cache disabled. This will remove the pulled or build images
           and start building images from scratch. This might take a long time.

diff --git a/Dockerfile b/Dockerfile
@@ -121,29 +121,11 @@ RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-# Install MySQL client from Oracle repositories (Debian installs mariadb)
-RUN KEY="A4A9406876FCBD3C456770C88C718D3B5072E1F5" \
-    && GNUPGHOME="$(mktemp -d)" \
-    && export GNUPGHOME \
-    && for KEYSERVER in $(shuf -e \
-            ha.pool.sks-keyservers.net \
-            hkp://p80.pool.sks-keyservers.net:80 \
-            keyserver.ubuntu.com \
-            hkp://keyserver.ubuntu.com:80 \
-            pgp.mit.edu) ; do \
-          gpg --keyserver "${KEYSERVER}" --recv-keys "${KEY}" && break || true ; \
-       done \
-    && gpg --export "${KEY}" | apt-key add - \
-    && gpgconf --kill all \
-    rm -rf "${GNUPGHOME}"; \
-    apt-key list > /dev/null \
-    && echo "deb http://repo.mysql.com/apt/debian/ stretch mysql-5.7" | tee -a /etc/apt/sources.list.d/mysql.list \
-    && apt-get update \
-    && apt-get install --no-install-recommends -y \
-        libmysqlclient-dev \
-        mysql-client \
-    && apt-get autoremove -yqq --purge \
-    && apt-get clean && rm -rf /var/lib/apt/lists/*
+ARG INSTALL_MYSQL_CLIENT="true"
+ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT}
+
+COPY scripts/docker scripts/docker
+RUN ./scripts/docker/install_mysql.sh dev
 
 ARG AIRFLOW_REPO=apache/airflow
 ENV AIRFLOW_REPO=${AIRFLOW_REPO}
@@ -168,9 +150,12 @@ ENV AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES}
 # In case of Production build image segment we want to pre-install master version of airflow
 # dependencies from github so that we do not have to always reinstall it from the scratch.
 RUN if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" ]]; then \
-        pip install --user \
-            "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
-                --constraint "${AIRFLOW_CONSTRAINTS_URL}" && pip uninstall --yes apache-airflow; \
+       if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then \
+          AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,}; \
+       fi; \
+       pip install --user \
+          "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
+          --constraint "${AIRFLOW_CONSTRAINTS_URL}" && pip uninstall --yes apache-airflow; \
     fi
 
 ARG AIRFLOW_SOURCES_FROM="."
@@ -201,7 +186,11 @@ ENV SLUGIFY_USES_TEXT_UNIDECODE=${SLUGIFY_USES_TEXT_UNIDECODE}
 
 WORKDIR /opt/airflow
 
-RUN pip install --user "${AIRFLOW_INSTALL_SOURCES}[${AIRFLOW_EXTRAS}]${AIRFLOW_INSTALL_VERSION}" \
+# remove mysql from extras if client is not installed
+RUN if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then \
+        AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,}; \
+    fi; \
+    pip install --user "${AIRFLOW_INSTALL_SOURCES}[${AIRFLOW_EXTRAS}]${AIRFLOW_INSTALL_VERSION}" \
     --constraint "${AIRFLOW_CONSTRAINTS_URL}" && \
     if [ -n "${ADDITIONAL_PYTHON_DEPS}" ]; then pip install --user ${ADDITIONAL_PYTHON_DEPS} \
     --constraint "${AIRFLOW_CONSTRAINTS_URL}"; fi && \
@@ -306,29 +295,11 @@ RUN mkdir -pv /usr/share/man/man1 \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-# Install MySQL client from Oracle repositories (Debian installs mariadb)
-RUN KEY="A4A9406876FCBD3C456770C88C718D3B5072E1F5" \
-    && GNUPGHOME="$(mktemp -d)" \
-    && export GNUPGHOME \
-    && for KEYSERVER in $(shuf -e \
-            ha.pool.sks-keyservers.net \
-            hkp://p80.pool.sks-keyservers.net:80 \
-            keyserver.ubuntu.com \
-            hkp://keyserver.ubuntu.com:80 \
-            pgp.mit.edu) ; do \
-          gpg --keyserver "${KEYSERVER}" --recv-keys "${KEY}" && break || true ; \
-       done \
-    && gpg --export "${KEY}" | apt-key add - \
-    && gpgconf --kill all \
-    rm -rf "${GNUPGHOME}"; \
-    apt-key list > /dev/null \
-    && echo "deb http://repo.mysql.com/apt/debian/ stretch mysql-5.7" | tee -a /etc/apt/sources.list.d/mysql.list \
-    && apt-get update \
-    && apt-get install --no-install-recommends -y \
-        libmysqlclient20 \
-        mysql-client \
-    && apt-get autoremove -yqq --purge \
-    && apt-get clean && rm -rf /var/lib/apt/lists/*
+ARG INSTALL_MYSQL_CLIENT="true"
+ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT}
+
+COPY scripts/docker scripts/docker
+RUN ./scripts/docker/install_mysql.sh prod
 
 ENV AIRFLOW_UID=${AIRFLOW_UID}
 ENV AIRFLOW_GID=${AIRFLOW_GID}

diff --git a/Dockerfile.ci b/Dockerfile.ci
@@ -91,29 +91,8 @@ RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-# Install MySQL client from Oracle repositories (Debian installs mariadb)
-RUN KEY="A4A9406876FCBD3C456770C88C718D3B5072E1F5" \
-    && GNUPGHOME="$(mktemp -d)" \
-    && export GNUPGHOME \
-    && for KEYSERVER in $(shuf -e \
-            ha.pool.sks-keyservers.net \
-            hkp://p80.pool.sks-keyservers.net:80 \
-            keyserver.ubuntu.com \
-            hkp://keyserver.ubuntu.com:80 \
-            pgp.mit.edu) ; do \
-          gpg --keyserver "${KEYSERVER}" --recv-keys "${KEY}" && break || true ; \
-       done \
-    && gpg --export "${KEY}" | apt-key add - \
-    && gpgconf --kill all \
-    rm -rf "${GNUPGHOME}"; \
-    apt-key list > /dev/null \
-    && echo "deb http://repo.mysql.com/apt/debian/ stretch mysql-5.7" | tee -a /etc/apt/sources.list.d/mysql.list \
-    && apt-get update \
-    && apt-get install --no-install-recommends -y \
-        libmysqlclient-dev \
-        mysql-client \
-    && apt-get autoremove -yqq --purge \
-    && apt-get clean && rm -rf /var/lib/apt/lists/*
+COPY scripts/docker scripts/docker
+RUN ./scripts/docker/install_mysql.sh dev
 
 RUN adduser airflow \
     && echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \

diff --git a/breeze b/breeze
@@ -955,6 +955,11 @@ function breeze::parse_arguments() {
             echo "Additional apt runtime dependencies: ${ADDITIONAL_RUNTIME_DEPS}"
             shift 2
             ;;
+        --disable-mysql-client-installation)
+            export INSTALL_MYSQL_CLIENT="false"
+            echo "Install MySQL client: ${INSTALL_MYSQL_CLIENT}"
+            shift
+            ;;
         -D | --dockerhub-user)
             export DOCKERHUB_USER="${2}"
             echo "Dockerhub user ${DOCKERHUB_USER}"
@@ -2224,6 +2229,10 @@ ${FORMATTED_DEFAULT_PROD_EXTRAS}
 --disable-pip-cache
         Disables GitHub PIP cache during the build. Useful if github is not reachable during build.
 
+--disable-mysql-client-installation
+        Disables installation of the mysql client which might be problematic if you are building
+        image in controlled environment. Only valid for production image.
+
 -C, --force-clean-images
         Force build images with cache disabled. This will remove the pulled or build images
         and start building images from scratch. This might take a long time.

diff --git a/breeze-complete b/breeze-complete
@@ -149,6 +149,7 @@ dockerhub-user: dockerhub-repo: github-registry github-repository: github-image-
 postgres-version: mysql-version:
 version-suffix-for-pypi: version-suffix-for-svn:
 additional-extras: additional-python-deps: additional-dev-deps: additional-runtime-deps:
+disable-mysql-client-installation
 load-default-connections load-example-dags
 "
 

diff --git a/docs/production-deployment.rst b/docs/production-deployment.rst
@@ -326,6 +326,10 @@ The following build arguments (``--build-arg`` in docker build command) can be u
 |                                          |                                          | installing in case cassandra extra is    |
 |                                          |                                          | used).                                   |
 +------------------------------------------+------------------------------------------+------------------------------------------+
+| ``INSTALL_MYSQL_CLIENT``                 | ``true``                                 | Whether MySQL client should be installed |
+|                                          |                                          | The mysql extra is removed from extras   |
+|                                          |                                          | if the client is not installed           |
++------------------------------------------+------------------------------------------+------------------------------------------+
 
 There are build arguments that determine the installation mechanism of Apache Airflow for the
 production image. There are three types of build:

diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh
@@ -684,6 +684,7 @@ function build_images::build_prod_images() {
         "${EXTRA_DOCKER_PROD_BUILD_FLAGS[@]}" \
         --build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE}" \
         --build-arg PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}" \
+        --build-arg INSTALL_MYSQL_CLIENT="${INSTALL_MYSQL_CLIENT}" \
         --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
         --build-arg AIRFLOW_BRANCH="${AIRFLOW_BRANCH_FOR_PYPI_PRELOADING}" \
         --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \
@@ -701,6 +702,7 @@ function build_images::build_prod_images() {
         "${EXTRA_DOCKER_PROD_BUILD_FLAGS[@]}" \
         --build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE}" \
         --build-arg PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}" \
+        --build-arg INSTALL_MYSQL_CLIENT="${INSTALL_MYSQL_CLIENT}" \
         --build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \
         --build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \
         --build-arg ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS}" \

diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
@@ -216,9 +216,7 @@ function initialization::initialize_mount_variables() {
         verbosity::print_info
         verbosity::print_info "Mounting files folder to Docker"
         verbosity::print_info
-        EXTRA_DOCKER_FLAGS+=(
-            "-v" "${AIRFLOW_SOURCES}/files:/files"
-        )
+        EXTRA_DOCKER_FLAGS+=("-v" "${AIRFLOW_SOURCES}/files:/files")
     fi
 
     EXTRA_DOCKER_FLAGS+=(
@@ -322,6 +320,8 @@ function initialization::initialize_image_build_variables() {
     export ADDITIONAL_RUNTIME_DEPS="${ADDITIONAL_RUNTIME_DEPS:=""}"
     # whether pre cached pip packages are used during build
     export AIRFLOW_PRE_CACHED_PIP_PACKAGES="${AIRFLOW_PRE_CACHED_PIP_PACKAGES:="true"}"
+    # by default install mysql client
+    export INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT:="true"}
 }
 
 # Determine version suffixes used to build backport packages

diff --git a/scripts/docker/install_mysql.sh b/scripts/docker/install_mysql.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+declare -a packages
+
+if [[ "${1}" == "dev" ]]; then
+    packages=("libmysqlclient-dev" "mysql-client")
+elif [[ "${1}" == "prod" ]]; then
+    packages=("libmysqlclient20" "mysql-client")
+else
+    echo
+    echo "Specify either prod or dev"
+    echo
+fi
+
+# Install MySQL Client during the container build
+set -euo pipefail
+
+# Install MySQL client from Oracle repositories (Debian installs mariadb)
+# But only if it is not disabled
+if [[ ${INSTALL_MYSQL_CLIENT:="true"} == "true" ]]; then
+    KEY="A4A9406876FCBD3C456770C88C718D3B5072E1F5"
+    readonly KEY
+
+    GNUPGHOME="$(mktemp -d)"
+    export GNUPGHOME
+    set +e
+    for keyserver in $(shuf -e ha.pool.sks-keyservers.net hkp://p80.pool.sks-keyservers.net:80 \
+                               keyserver.ubuntu.com hkp://keyserver.ubuntu.com:80)
+    do
+        gpg --keyserver "${keyserver}" --recv-keys "${KEY}" && break
+    done
+    set -e
+    gpg --export "${KEY}" | apt-key add -
+    gpgconf --kill all
+    rm -rf "${GNUPGHOME}"
+    apt-key list > /dev/null
+    echo "deb http://repo.mysql.com/apt/debian/ stretch mysql-5.7" | tee -a /etc/apt/sources.list.d/mysql.list
+    apt-get update
+    apt-get install --no-install-recommends -y "${packages[@]}"
+    apt-get autoremove -yqq --purge
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+fi