Skip to content

Commit

Permalink
Add extras when installing prod image from packages (apache#13432)
Browse files Browse the repository at this point in the history 
In the latest change apache#13422 change in the way product images are
prepared removed extras from installed airflow - thus caused
failing production image verification check.

This change restores extras when airflow is installed from packages
  • Loading branch information
@potiuk
potiuk authored Jan 2, 2021
1 parent 0909ddf commit 3a73110
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 15 deletions.
36 changes: 28 additions & 8 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -265,20 +265,37 @@ RUN if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then \
fi; \
fi; \
if [[ ${INSTALL_FROM_DOCKER_CONTEXT_FILES} == "true" ]]; then \
reinstalling_apache_airflow_packages=$(ls /docker-context-files/apache?airflow*.{whl,tar.gz} 2>/dev/null || true); \
# We want to install apache airflow packages with constraints \
if [[ "${reinstalling_apache_airflow_packages}" != "" ]]; then \
# We want to install apache airflow package with extras - that's why it is a separate step \
# But we should still install them with all dependencies \
reinstalling_apache_airflow_package=$(ls /docker-context-files/apache?airflow?[0-9]*.{whl,tar.gz} 2>/dev/null || true); \
if [[ "${reinstalling_apache_airflow_package}" != "" ]]; then \
if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \
pip install --force-reinstall --upgrade --upgrade-strategy eager \
--user ${reinstalling_apache_airflow_packages}; \
--user "${reinstalling_apache_airflow_package}[${AIRFLOW_EXTRAS}]"; \
pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
else \
# We want to install apache airflow package with constraints \
pip install --force-reinstall --upgrade --upgrade-strategy only-if-needed \
--user ${reinstalling_apache_airflow_packages} --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
--user "${reinstalling_apache_airflow_package}[${AIRFLOW_EXTRAS}]" --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
fi; \
fi ; \
# All the others we want to reinstall as-is, without dependencies \
reinstalling_apache_airflow_providers_packages=$(ls /docker-context-files/apache?airflow?providers*.{whl,tar.gz} 2>/dev/null || true); \
# We want to install apache airflow package with extras - that's why it is a separate step \
# But we should still install them with all dependencies \
if [[ "${reinstalling_apache_airflow_providers_packages}" != "" ]]; then \
if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \
pip install --force-reinstall --upgrade --upgrade-strategy eager \
--user ${reinstalling_apache_airflow_providers_packages}; \
pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
else \
# We want to install apache airflow provider packages with constraints \
pip install --force-reinstall --upgrade --upgrade-strategy only-if-needed \
--user ${reinstalling_apache_airflow_providers_packages} --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
fi; \
fi ; \
# All the other packages we want to reinstall as-is, without dependencies \
reinstalling_other_packages=$(ls /docker-context-files/*.{whl,tar.gz} 2>/dev/null | \
grep -v apache_airflow | grep -v apache-airflow || true); \
if [[ "${reinstalling_other_packages}" != "" ]]; then \
Expand All @@ -287,8 +304,11 @@ RUN if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then \
fi; \
if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \
if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \
pip install --user ${ADDITIONAL_PYTHON_DEPS} --upgrade --upgrade-strategy eager; \
pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
pip install --user ${ADDITIONAL_PYTHON_DEPS} \
${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} \
--upgrade --upgrade-strategy eager; \
pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE}; \
else \
pip install --user ${ADDITIONAL_PYTHON_DEPS} --upgrade --upgrade-strategy only-if-needed \
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
Expand Down
27 changes: 20 additions & 7 deletions Dockerfile.ci
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,26 +337,39 @@ COPY docker-context-files/ /docker-context-files/

# hadolint ignore=SC2086, SC2010
RUN if [[ ${INSTALL_FROM_DOCKER_CONTEXT_FILES} == "true" ]]; then \
reinstalling_apache_airflow_packages=$(ls /docker-context-files/apache?airflow*.{whl,tar.gz} 2>/dev/null || true); \
# We want to install apache airflow packages with constraints \
if [[ "${reinstalling_apache_airflow_packages}" != "" ]]; then \
reinstalling_apache_airflow_package=$(ls /docker-context-files/apache?airflow?[0-9]*.{whl,tar.gz} 2>/dev/null || true); \
# We want to install apache airflow package with constraints \
if [[ "${reinstalling_apache_airflow_package}" != "" ]]; then \
if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \
pip install --force-reinstall --upgrade --upgrade-strategy eager \
--user ${reinstalling_apache_airflow_packages}; \
--user ${reinstalling_apache_airflow_package}[${AIRFLOW_EXTRAS}]; \
pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
else \
pip install --force-reinstall --upgrade --upgrade-strategy only-if-needed \
--user ${reinstalling_apache_airflow_packages} --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
--user ${reinstalling_apache_airflow_package}[${AIRFLOW_EXTRAS}] --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
fi; \
fi ; \
# All the others we want to reinstall as-is, without dependencies \
reinstalling_apache_airflow_providers_packages=$(ls /docker-context-files/apache?airflow?providers*.{whl,tar.gz} 2>/dev/null || true); \
# We want to install apache airflow provider packages with constraints \
if [[ "${reinstalling_apache_airflow_providers_packages}" != "" ]]; then \
if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \
pip install --force-reinstall --upgrade --upgrade-strategy eager \
--user ${reinstalling_apache_airflow_providers_packages}; \
pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
else \
pip install --force-reinstall --upgrade --upgrade-strategy only-if-needed \
--user ${reinstalling_apache_airflow_providers_packages} --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
fi; \
fi ; \
# All the other packages we want to reinstall as-is, without dependencies \
reinstalling_other_packages=$(ls /docker-context-files/*.{whl,tar.gz} 2>/dev/null | \
grep -v apache_airflow | grep -v apache-airflow || true); \
if [[ "${reinstalling_other_packages}" != "" ]]; then \
pip install --force-reinstall --user --no-deps ${reinstalling_other_packages}; \
fi; \
fi;
fi

# Copy all the www/ files we need to compile assets. Done as two separate COPY
# commands so as otherwise it copies the _contents_ of static/ in to www/
Expand Down

0 comments on commit 3a73110

Please sign in to comment.