Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SC-XXX: Improved speed of docker image building #449

Open
wants to merge 26 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
8e8689b
Improved speed of docker image building
sprymiker Jun 28, 2023
252ff73
Moved dockerfiles to match current structure and keep git history as …
sprymiker Jun 28, 2023
6e1a6e0
Adjusted default progress type to auto for more compatibility
sprymiker Jun 28, 2023
1c9721c
Replaced dependency on `ip` package in linux to use more common packages
sprymiker Jun 28, 2023
fcecee2
Fixed wrong path appeared after merging with master
sprymiker Jun 28, 2023
59d5a9f
Adjusted bake configuration by separating targets in export process a…
sprymiker Jun 29, 2023
c0bc1ba
- Added fallback for cases when buildx docker plugin is not installed
sprymiker Jun 29, 2023
8c4fe3c
- Reduced requirement for buildx version as it is confirmed working o…
sprymiker Jun 29, 2023
ba082cb
- Fixed mount cache permissions and wrong id
sprymiker Jun 30, 2023
a69258f
Fixed git cleanup command (mostly for nonsplit)
sprymiker Jun 30, 2023
31715fb
Fixed git cleanup command (mostly for nonsplit). Part 2
sprymiker Jun 30, 2023
1787de8
Deep speed, number of layers and image size optimizations. Restructur…
sprymiker Jul 2, 2023
f833e16
Speeding up dev composer run
sprymiker Jul 2, 2023
a418643
Fixed assets build process that does not build
sprymiker Jul 2, 2023
728d8dd
Excluding tests/dd.php to be skipped for web app images
sprymiker Jul 2, 2023
c8dcc4b
Reduced number of layers in web service images
sprymiker Jul 3, 2023
ff820d6
Fixed autoload issue with spryker/spryker
sprymiker Jul 3, 2023
1ec23b4
Reduced pipeline image by excluding git folders in vendor, but keepin…
sprymiker Jul 3, 2023
5e7cb85
Fixed resolver cache for cli/pipeline
sprymiker Jul 3, 2023
293a8fd
Fixed cli and scheduler does not work properly
sprymiker Jul 3, 2023
a91406d
Fixed permission issue in cli
sprymiker Jul 3, 2023
d321fb0
Optimized pipeline build speed by reusing application build results
sprymiker Jul 3, 2023
2f22727
Optimized pipeline build speed by reusing application build results […
sprymiker Jul 3, 2023
8d7a792
Fixed lack of files in root folder for FE build in suite-nonsplit (an…
sprymiker Jul 4, 2023
dd734ce
Improved docker build structure and hopefully speed be using --mount=…
sprymiker Jul 4, 2023
a60b3fd
Refactored Dockerfiles to use newest build features
sprymiker Aug 3, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
- Added fallback for cases when buildx docker plugin is not installed
- Removed intermediate dump-autoload as it probably takes more time than benifits for the following build
  • Loading branch information
sprymiker committed Jun 29, 2023
commit c0bc1baad21f3f469b7048bb845cd59d118a41ee
22 changes: 19 additions & 3 deletions bin/sdk/images/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,29 @@ function Images::destroy() {
docker rmi -f "${SPRYKER_PLATFORM_IMAGE}" 2>/dev/null || true
}

function Images::_checkBuildxVersion() {
min_version=${1}
version_regex="v([0-9]+\.[0-9]+\.[0-9]+)"
actual_version=$([[ $(docker buildx version) =~ $version_regex ]] && echo "${BASH_REMATCH[1]}")
greater_version=$(printf "%s\n%s\n" "${actual_version}" "${min_version}" | sort -t '.' -k 1,1 -k 2,2 -k 3,3 -g | tail -n 1)
if [ "$min_version" == "$greater_version" ]; then
return "${FALSE}"
fi
}

function Images::_build() {

# Checking availability of docker bake
if docker buildx --help | grep bake >/dev/null 2>&1; then
import sdk/images/engine/bake.sh
# Checking availability of docker bake or buildx
if docker buildx >/dev/null 2>&1; then
if Images::_checkBuildxVersion "0.8.99"; then
import sdk/images/engine/bake.sh
else
import sdk/images/engine/buildx.sh
Console::warn 'Warning! Upgrade `buildx` docker plugin to the latest version for better performance'
fi
else
import sdk/images/engine/build.sh
Console::warn 'Warning! Install `buildx` docker plugin for better performance'
fi

# Using temporary file for secrets as `docker secret` is only available for swarm mode.
Expand Down
91 changes: 91 additions & 0 deletions bin/sdk/images/engine/build.common.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
#!/bin/bash

require docker

function Images::_build::perform() {

local folder=${1}
local TARGET_TAG=${2:-${SPRYKER_DOCKER_TAG}}
local destination=${3:-print}

local targetImage="${SPRYKER_DOCKER_PREFIX}_target:${SPRYKER_DOCKER_TAG}"

Console::verbose "${INFO}Building images${NC}"

# Primal build including all stages to fully benefit from multistage features
Images::build::_runBuild --target "target" -t "${targetImage}"

# Tagging all the images by running the same build targeting different stages
source ${DEPLOYMENT_PATH}/images/${folder}/${destination}.sh

local -a arguments=('--quiet' '--progress' 'auto')

if [ -n "${SSH_AUTH_SOCK_IN_CLI}" ]; then
arguments+=('--ssh' 'default')
fi

if Images::needPush; then
Console::verbose "${INFO}Tagging and pushing images${NC}"
else
Console::verbose "${INFO}Tagging images${NC}"
fi

arguments+=($(Images::_build::prepareArguments))

local -a pids
local targetData
for targetData in "${TARGET_TAGS[@]}"; do
eval "${targetData}"

local -a tagArguments=()
local tag
for tag in "${TAGS[@]}"; do
tagArguments+=('-t' "${tag}")
done

Console::info "${YELLOW}Target:${NC} ${TARGET}"
Console::verbose "${YELLOW}Tags:${NC}"
Console::verbose "${DGRAY}${TAGS[@]}${NC}"
Console::verbose "${YELLOW}Hash:${NC}"
Images::build::_runBuild --target "${TARGET}" "${arguments[@]}" "${tagArguments[@]}"

Images::_build::afterTaggingAnImage "${TAGS[@]}" &
pids+=($!)
done

docker rmi -f "${targetImage}" >/dev/null 2>&1 || true

local pid
for pid in "$pids[@]"; do
wait $pid || exit 1
done
}

function Images::build::_runBuild {

docker build \
-f "${DEPLOYMENT_PATH}/images/${folder}/Dockerfile" \
"${@}" \
--secret "id=secrets-env,src=$SECRETS_FILE_PATH" \
--label "spryker.project=${SPRYKER_DOCKER_PREFIX}" \
--label "spryker.revision=${SPRYKER_BUILD_HASH}" \
--label "spryker.sdk.revision=${SPRYKER_SDK_REVISION}" \
--build-arg "DEPLOYMENT_PATH=${DEPLOYMENT_PATH}" \
--build-arg "SPRYKER_PLATFORM_IMAGE=${SPRYKER_PLATFORM_IMAGE}" \
--build-arg "SPRYKER_FRONTEND_IMAGE=${SPRYKER_FRONTEND_IMAGE}" \
--build-arg "SPRYKER_LOG_DIRECTORY=${SPRYKER_LOG_DIRECTORY}" \
--build-arg "SPRYKER_PIPELINE=${SPRYKER_PIPELINE}" \
--build-arg "APPLICATION_ENV=${APPLICATION_ENV}" \
--build-arg "SPRYKER_COMPOSER_MODE=${SPRYKER_COMPOSER_MODE}" \
--build-arg "SPRYKER_COMPOSER_AUTOLOAD=${SPRYKER_COMPOSER_AUTOLOAD}" \
--build-arg "SPRYKER_ASSETS_MODE=${SPRYKER_ASSETS_MODE}" \
--build-arg "SPRYKER_DB_ENGINE=${SPRYKER_DB_ENGINE}" \
--build-arg "KNOWN_HOSTS=${KNOWN_HOSTS}" \
--build-arg "SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH}" \
--build-arg "SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP}" \
--build-arg "SPRYKER_NODE_IMAGE_VERSION=${SPRYKER_NODE_IMAGE_VERSION}" \
--build-arg "SPRYKER_NODE_IMAGE_DISTRO=${SPRYKER_NODE_IMAGE_DISTRO}" \
--build-arg "SPRYKER_NPM_VERSION=${SPRYKER_NPM_VERSION}" \
--build-arg "USER_UID=${USER_UID}" \
./ >&2
}
83 changes: 10 additions & 73 deletions bin/sdk/images/engine/build.sh
Original file line number Diff line number Diff line change
@@ -1,82 +1,19 @@
#!/bin/bash

require docker
import sdk/images/engine/build.common.sh

function Images::_build::perform() {

local folder=${1}
local TARGET_TAG=${2:-${SPRYKER_DOCKER_TAG}}
local destination=${3:-print}

local targetImage="${SPRYKER_DOCKER_PREFIX}_target:${SPRYKER_DOCKER_TAG}"

Console::verbose "${INFO}Building images${NC}"

# Primal build including all stages to fully benefit from multistage features
Images::build::_runBuild --target "target" -t "${targetImage}"

# Tagging all the images by running the same build targeting different stages
source ${DEPLOYMENT_PATH}/images/${folder}/${destination}.sh

local -a arguments=('--quiet' '--progress' 'auto')
function Images::_build::prepareArguments() {
:
}

if [ -n "${SSH_AUTH_SOCK_IN_CLI}" ]; then
arguments+=('--ssh' 'default')
fi
function Images::_build::afterTaggingAnImage() {
set -x
local -a tagsToPush=("${@}")

if Images::needPush; then
arguments+=('--push')
Console::verbose "${INFO}Tagging and pushing images${NC}"
else
Console::verbose "${INFO}Tagging images${NC}"
fi

local targetData
for targetData in "${TARGET_TAGS[@]}"; do
eval "${targetData}"

local -a tagArguments=()
local tag
for tag in "${TAGS[@]}"; do
tagArguments+=('-t' "${tag}")
for tag in "${tagsToPush[@]}"; do
docker push ${tag}
done

Console::info "${YELLOW}Target:${NC} ${TARGET}"
Console::verbose "${YELLOW}Tags:${NC}"
Console::verbose "${DGRAY}${TAGS[@]}${NC}"
Console::verbose "${YELLOW}Hash:${NC}"
Images::build::_runBuild --target "${TARGET}" "${arguments[@]}" "${tagArguments[@]}"

done

docker rmi -f "${targetImage}" >/dev/null 2>&1 || true
}

function Images::build::_runBuild {

docker build \
-f "${DEPLOYMENT_PATH}/images/${folder}/Dockerfile" \
"${@}" \
--secret "id=secrets-env,src=$SECRETS_FILE_PATH" \
--label "spryker.project=${SPRYKER_DOCKER_PREFIX}" \
--label "spryker.revision=${SPRYKER_BUILD_HASH}" \
--label "spryker.sdk.revision=${SPRYKER_SDK_REVISION}" \
--build-arg "DEPLOYMENT_PATH=${DEPLOYMENT_PATH}" \
--build-arg "SPRYKER_PLATFORM_IMAGE=${SPRYKER_PLATFORM_IMAGE}" \
--build-arg "SPRYKER_FRONTEND_IMAGE=${SPRYKER_FRONTEND_IMAGE}" \
--build-arg "SPRYKER_LOG_DIRECTORY=${SPRYKER_LOG_DIRECTORY}" \
--build-arg "SPRYKER_PIPELINE=${SPRYKER_PIPELINE}" \
--build-arg "APPLICATION_ENV=${APPLICATION_ENV}" \
--build-arg "SPRYKER_COMPOSER_MODE=${SPRYKER_COMPOSER_MODE}" \
--build-arg "SPRYKER_COMPOSER_AUTOLOAD=${SPRYKER_COMPOSER_AUTOLOAD}" \
--build-arg "SPRYKER_ASSETS_MODE=${SPRYKER_ASSETS_MODE}" \
--build-arg "SPRYKER_DB_ENGINE=${SPRYKER_DB_ENGINE}" \
--build-arg "KNOWN_HOSTS=${KNOWN_HOSTS}" \
--build-arg "SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH}" \
--build-arg "SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP}" \
--build-arg "SPRYKER_NODE_IMAGE_VERSION=${SPRYKER_NODE_IMAGE_VERSION}" \
--build-arg "SPRYKER_NODE_IMAGE_DISTRO=${SPRYKER_NODE_IMAGE_DISTRO}" \
--build-arg "SPRYKER_NPM_VERSION=${SPRYKER_NPM_VERSION}" \
--build-arg "USER_UID=${USER_UID}" \
./ >&2
fi
}
13 changes: 13 additions & 0 deletions bin/sdk/images/engine/buildx.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
#!/bin/bash

import sdk/images/engine/build.common.sh

function Images::_build::prepareArguments() {
if Images::needPush; then
echo '--push'
fi
}

function Images::_build::afterTaggingAnImage() {
:
}
2 changes: 0 additions & 2 deletions images/templates/baked/application.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,6 @@ LABEL "spryker.image" "none"

USER spryker

RUN composer dump-autoload -o

ARG SPRYKER_PIPELINE
ENV SPRYKER_PIPELINE=${SPRYKER_PIPELINE}
ARG SPRYKER_DB_ENGINE
Expand Down