Upgrade Postgres client in Docker images (apache#21631)

Dockerfile

@@ -87,7 +87,6 @@ ARG DEV_APT_DEPS="\
      libffi-dev \
      libkrb5-dev \
      libldap2-dev \
-     libpq-dev \
      libsasl2-2 \
      libsasl2-dev \
      libsasl2-modules \
@@ -96,7 +95,6 @@ ARG DEV_APT_DEPS="\
      lsb-release \
      nodejs \
      openssh-client \
-     postgresql-client \
      python-selinux \
      sasl2-bin \
      software-properties-common \
@@ -143,6 +141,7 @@ RUN apt-get update \
 
 ARG INSTALL_MYSQL_CLIENT="true"
 ARG INSTALL_MSSQL_CLIENT="true"
+ARG INSTALL_POSTGRES_CLIENT="true"
 ARG AIRFLOW_REPO=apache/airflow
 ARG AIRFLOW_BRANCH=main
 ARG AIRFLOW_EXTRAS
@@ -195,13 +194,16 @@ ARG AIRFLOW_USER_HOME_DIR
 ARG AIRFLOW_UID
 
 ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} \
-    INSTALL_MSSQL_CLIENT=${INSTALL_MSSQL_CLIENT}
+    INSTALL_MSSQL_CLIENT=${INSTALL_MSSQL_CLIENT} \
+    INSTALL_POSTGRES_CLIENT=${INSTALL_POSTGRES_CLIENT}
 
 # Only copy mysql/mssql installation scripts for now - so that changing the other
 # scripts which are needed much later will not invalidate the docker layer here
-COPY scripts/docker/install_mysql.sh scripts/docker/install_mssql.sh /scripts/docker/
+COPY scripts/docker/install_mysql.sh scripts/docker/install_mssql.sh scripts/docker/install_postgres.sh /scripts/docker/
 
-RUN bash /scripts/docker/install_mysql.sh dev && bash /scripts/docker/install_mssql.sh
+RUN bash /scripts/docker/install_mysql.sh dev && \
+    bash /scripts/docker/install_mssql.sh && \
+    bash /scripts/docker/install_postgres.sh dev
 ENV PATH=${PATH}:/opt/mssql-tools/bin
 
 COPY docker-context-files /docker-context-files
@@ -382,7 +384,6 @@ ARG RUNTIME_APT_DEPS="\
        lsb-release \
        netcat \
        openssh-client \
-       postgresql-client \
        rsync \
        sasl2-bin \
        sqlite3 \
@@ -394,13 +395,15 @@ ARG ADDITIONAL_RUNTIME_APT_COMMAND=""
 ARG ADDITIONAL_RUNTIME_APT_ENV=""
 ARG INSTALL_MYSQL_CLIENT="true"
 ARG INSTALL_MSSQL_CLIENT="true"
+ARG INSTALL_POSTGRES_CLIENT="true"
 
 ENV RUNTIME_APT_DEPS=${RUNTIME_APT_DEPS} \
     ADDITIONAL_RUNTIME_APT_DEPS=${ADDITIONAL_RUNTIME_APT_DEPS} \
     RUNTIME_APT_COMMAND=${RUNTIME_APT_COMMAND} \
     ADDITIONAL_RUNTIME_APT_COMMAND=${ADDITIONAL_RUNTIME_APT_COMMAND} \
     INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} \
     INSTALL_MSSQL_CLIENT=${INSTALL_MSSQL_CLIENT} \
+    INSTALL_POSTGRES_CLIENT=${INSTALL_POSTGRES_CLIENT} \
     GUNICORN_CMD_ARGS="--worker-tmp-dir /dev/shm" \
     AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD}
 
@@ -440,13 +443,14 @@ ENV PATH="${AIRFLOW_USER_HOME_DIR}/.local/bin:${PATH}" \
 
 # Only copy mysql/mssql installation scripts for now - so that changing the other
 # scripts which are needed much later will not invalidate the docker layer here.
-COPY scripts/docker/install_mysql.sh /scripts/docker/install_mssql.sh /scripts/docker/
+COPY scripts/docker/install_mysql.sh /scripts/docker/install_mssql.sh /scripts/docker/install_postgres.sh /scripts/docker/
 # We run scripts with bash here to make sure we can execute the scripts. Changing to +x might have an
 # unexpected result - the cache for Dockerfiles might get invalidated in case the host system
 # had different umask set and group x bit was not set. In Azure the bit might be not set at all.
 # That also protects against AUFS Docker backen dproblem where changing the executable bit required sync
 RUN bash /scripts/docker/install_mysql.sh prod \
     && bash /scripts/docker/install_mssql.sh \
+    && bash /scripts/docker/install_postgres.sh prod \
     && adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password \
            --quiet "airflow" --uid "${AIRFLOW_UID}" --gid "0" --home "${AIRFLOW_USER_HOME_DIR}" \
 # Make Airflow files belong to the root group and are accessible. This is to accommodate the guidelines from

Dockerfile.ci

@@ -34,7 +34,8 @@ ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \
     LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 \
     DEPENDENCIES_EPOCH_NUMBER=${DEPENDENCIES_EPOCH_NUMBER} \
     INSTALL_MYSQL_CLIENT="true" \
-    INSTALL_MSSQL_CLIENT="true"
+    INSTALL_MSSQL_CLIENT="true" \
+    INSTALL_POSTGRES_CLIENT="true"
 
 RUN echo "Base image version: ${PYTHON_BASE_IMAGE}"
 
@@ -95,14 +96,15 @@ RUN apt-get update \
 
 # Only copy mysql/mssql installation scripts for now - so that changing the other
 # scripts which are needed much later will not invalidate the docker layer here.
-COPY scripts/docker/install_mysql.sh scripts/docker/install_mssql.sh /scripts/docker/
+COPY scripts/docker/install_mysql.sh scripts/docker/install_mssql.sh scripts/docker/install_postgres.sh /scripts/docker/
 # We run scripts with bash here to make sure we can execute the scripts. Changing to +x might have an
 # unexpected result - the cache for Dockerfiles might get invalidated in case the host system
 # had different umask set and group x bit was not set. In Azure the bit might be not set at all.
 # That also protects against AUFS Docker backen dproblem where changing the executable bit required sync
-RUN bash /scripts/docker/install_mysql.sh prod  \
-    && bash /scripts/docker/install_mysql.sh dev  \
+RUN bash /scripts/docker/install_mysql.sh prod \
+    && bash /scripts/docker/install_mysql.sh dev \
     && bash /scripts/docker/install_mssql.sh \
+    && bash /scripts/docker/install_postgres.sh dev \
     && adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password \
               --quiet "airflow" --home "/home/airflow" \
     && echo -e "airflow\nairflow" | passwd airflow 2>&1 \
@@ -237,6 +239,7 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\
 # * install always current version of airflow
     INSTALL_MYSQL_CLIENT="true" \
     INSTALL_MSSQL_CLIENT="true" \
+    INSTALL_POSTGRES_CLIENT="true" \
     AIRFLOW_INSTALLATION_METHOD="." \
     AIRFLOW_INSTALL_EDITABLE_FLAG="--editable" \
     AIRFLOW_VERSION_SPECIFICATION="" \

breeze

@@ -1097,6 +1097,11 @@ function breeze::parse_arguments() {
             echo "Install MsSQL client: ${INSTALL_MSSQL_CLIENT}"
             shift
             ;;
+        --disable-postgres-client-installation)
+            export INSTALL_POSTGRES_CLIENT="false"
+            echo "Install Postgres client: ${INSTALL_POSTGRES_CLIENT}"
+            shift
+            ;;
         --constraints-location)
             export AIRFLOW_CONSTRAINTS_LOCATION="${2}"
             echo "Constraints location: ${AIRFLOW_CONSTRAINTS_LOCATION}"

docs/docker-stack/build-arg-ref.rst

@@ -159,6 +159,9 @@ for examples of using those arguments.
 +------------------------------------------+------------------------------------------+------------------------------------------+
 | ``INSTALL_MSSQL_CLIENT``                 | ``true``                                 | Whether MsSQL client should be installed |
 +------------------------------------------+------------------------------------------+------------------------------------------+
+| ``INSTALL_POSTGRES_CLIENT``              | ``true``                                 | Whether Postgres client should be        |
+|                                          |                                          | installed                                |
++------------------------------------------+------------------------------------------+------------------------------------------+
 
 Installing Airflow using different methods
 ..........................................

docs/docker-stack/build.rst

@@ -633,6 +633,8 @@ where you can build the image using the packages downloaded by passing those bui
   client from the Oracle repositories.
 * (Optional) ``INSTALL_MSSQL_CLIENT="false"`` if you do not want to install ``MsSQL``
   client from the Microsoft repositories.
+* (Optional) ``INSTALL_POSTGRES_CLIENT="false"`` if you do not want to install ``Postgres``
+  client from the Postgres repositories.
 
 Note, that the solution we have for installing python packages from local packages, only solves the problem
 of "air-gaped" python installation. The Docker image also downloads ``apt`` dependencies and ``node-modules``.

docs/docker-stack/docker-examples/restricted/restricted_environments.sh

@@ -43,6 +43,7 @@ docker build . \
     --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
     --build-arg INSTALL_MYSQL_CLIENT="false" \
     --build-arg INSTALL_MSSQL_CLIENT="false" \
+    --build-arg INSTALL_POSTGRES_CLIENT="true" \
     --build-arg AIRFLOW_PRE_CACHED_PIP_PACKAGES="false" \
     --build-arg INSTALL_FROM_DOCKER_CONTEXT_FILES="true" \
     --build-arg AIRFLOW_CONSTRAINTS_LOCATION="/docker-context-files/constraints-3.7.txt" \

scripts/ci/libraries/_build_images.sh

@@ -633,6 +633,7 @@ function build_images::build_prod_images() {
         --build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE}" \
         --build-arg INSTALL_MYSQL_CLIENT="${INSTALL_MYSQL_CLIENT}" \
         --build-arg INSTALL_MSSQL_CLIENT="${INSTALL_MSSQL_CLIENT}" \
+        --build-arg INSTALL_POSTGRES_CLIENT="${INSTALL_POSTGRES_CLIENT}" \
         --build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \
         --build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \
         --build-arg INSTALL_PROVIDERS_FROM_SOURCES="${INSTALL_PROVIDERS_FROM_SOURCES}" \

scripts/ci/libraries/_initialization.sh

@@ -232,6 +232,7 @@ function initialization::initialize_files_for_rebuild_check() {
         "scripts/docker/install_airflow_dependencies_from_branch_tip.sh"
         "scripts/docker/install_from_docker_context_files.sh"
         "scripts/docker/install_mysql.sh"
+        "scripts/docker/install_postgres.sh"
         "airflow/www/package.json"
         "airflow/www/yarn.lock"
         "airflow/www/webpack.config.js"
@@ -400,6 +401,8 @@ function initialization::initialize_image_build_variables() {
     export INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT:="true"}
     # by default install mssql client
     export INSTALL_MSSQL_CLIENT=${INSTALL_MSSQL_CLIENT:="true"}
+    # by default install postgres client
+    export INSTALL_POSTGRES_CLIENT=${INSTALL_POSTGRES_CLIENT:="true"}
     # additional tag for the image
     export IMAGE_TAG=${IMAGE_TAG:=""}
 

scripts/docker/install_airflow.sh

@@ -36,12 +36,18 @@ function install_airflow() {
           ${AIRFLOW_INSTALL_EDITABLE_FLAG} == "--editable" ]]; then
         echo
         echo "${COLOR_RED}ERROR! You can only use --editable flag when installing airflow from sources!${COLOR_RESET}"
-        echo "{COLOR_RED}       Current installation method is '${AIRFLOW_INSTALLATION_METHOD} and should be '.'${COLOR_RESET}"
+        echo "${COLOR_RED}       Current installation method is '${AIRFLOW_INSTALLATION_METHOD} and should be '.'${COLOR_RESET}"
         exit 1
     fi
     # Remove mysql from extras if client is not going to be installed
     if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then
         AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,}
+        echo "${COLOR_YELLOW}MYSQL client installation is disabled. Extra 'mysql' installations were therefore omitted.${COLOR_RESET}"
+    fi
+    # Remove postgres from extras if client is not going to be installed
+    if [[ ${INSTALL_POSTGRES_CLIENT} != "true" ]]; then
+        AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/postgres,}
+        echo "${COLOR_YELLOW}Postgres client installation is disabled. Extra 'postgres' installations were therefore omitted.${COLOR_RESET}"
     fi
     if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then
         echo

scripts/docker/install_airflow_dependencies_from_branch_tip.sh

@@ -24,13 +24,15 @@
 # deps from those pre-installed dependencies. It saves few minutes of build time when setup.py changes.
 #
 # If INSTALL_MYSQL_CLIENT is set to false, mysql extra is removed
+# If INSTALL_POSTGRES_CLIENT is set to false, postgres extra is removed
 #
 # shellcheck source=scripts/docker/common.sh
 . "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
 
 : "${AIRFLOW_REPO:?Should be set}"
 : "${AIRFLOW_BRANCH:?Should be set}"
 : "${INSTALL_MYSQL_CLIENT:?Should be true or false}"
+: "${INSTALL_POSTGRES_CLIENT:?Should be true or false}"
 : "${AIRFLOW_PIP_VERSION:?Should be set}"
 
 function install_airflow_dependencies_from_branch_tip() {
@@ -40,6 +42,9 @@ function install_airflow_dependencies_from_branch_tip() {
     if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then
        AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,}
     fi
+    if [[ ${INSTALL_POSTGRES_CLIENT} != "true" ]]; then
+       AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/postgres,}
+    fi
     # Install latest set of dependencies using constraints. In case constraints were upgraded and there
     # are conflicts, this might fail, but it should be fixed in the following installation steps
     pip install \

scripts/docker/install_from_docker_context_files.sh

@@ -30,6 +30,9 @@ function install_airflow_and_providers_from_docker_context_files(){
     if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then
         AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,}
     fi
+    if [[ ${INSTALL_POSTGRES_CLIENT} != "true" ]]; then
+        AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/postgres,}
+    fi
 
     # shellcheck disable=SC2206
     local pip_flags=(

scripts/docker/install_postgres.sh

@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# shellcheck shell=bash
+set -euo pipefail
+declare -a packages
+
+COLOR_BLUE=$'\e[34m'
+readonly COLOR_BLUE
+COLOR_RESET=$'\e[0m'
+readonly COLOR_RESET
+
+: "${INSTALL_POSTGRES_CLIENT:?Should be true or false}"
+
+install_postgres_client() {
+    echo
+    echo "${COLOR_BLUE}Installing postgres client${COLOR_RESET}"
+    echo
+
+    if [[ "${1}" == "dev" ]]; then
+        packages=("libpq-dev" "postgresql-client")
+    elif [[ "${1}" == "prod" ]]; then
+        packages=("postgresql-client")
+    else
+        echo
+        echo "Specify either prod or dev"
+        echo
+        exit 1
+    fi
+
+    curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+    echo "deb https://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list
+    apt-get update
+    apt-get install --no-install-recommends -y "${packages[@]}"
+    apt-get autoremove -yqq --purge
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+}
+
+# Install Postgres client from Postgres repositories
+# But only if it is not disabled
+if [[ ${INSTALL_POSTGRES_CLIENT:="true"} == "true" ]]; then
+    install_postgres_client "${@}"
+fi