docker: use buildkit cache mounts

The "docker build --network" parameter no longer accepts general networks when using BuildKit, and BuildKit has been the default for all platforms except Windows since Docker 23.0. One value that is still accepted by "docker build --network" is "host", so use the host network when building images, and run the rsync cache container on the host network as well. Being able to use BuildKit allows us to use cache mounts for ccache, so the second consecutive invocation of docker build on a machine will already have a warm cache and rsync will not copy anything.
huchim · Oct 8, 2024 · a822875 · a822875
1 parent 7a65292
commit a822875
Show file tree

Hide file tree

Showing 7 changed files with 40 additions and 54 deletions.
diff --git a/docker/alpine-normal/Dockerfile b/docker/alpine-normal/Dockerfile
@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:1
+
 ##
 # osgeo/gdal:alpine-normal
 
@@ -11,6 +13,8 @@ FROM alpine:${ALPINE_VERSION} AS builder
 # Derived from osgeo/proj by Howard Butler <[email protected]>
 LABEL maintainer="Even Rouault <[email protected]>"
 
+ENV HOME="/root"
+
 # Setup build env for PROJ
 RUN apk add --no-cache  wget curl unzip cmake make libtool autoconf automake pkgconfig gcc g++ sqlite sqlite-dev tiff-dev
 
@@ -141,13 +145,13 @@ ARG RSYNC_REMOTE
 
 # Build PROJ
 ARG PROJ_VERSION=master
-RUN mkdir proj \
+RUN --mount=type=cache,id=alpine-normal-proj,target=$HOME/.cache \
+    mkdir proj \
     && curl -L -fsS https://github.com/OSGeo/PROJ/archive/${PROJ_VERSION}.tar.gz \
         | tar xz -C proj --strip-components=1 \
     && cd proj \
     && if test "${RSYNC_REMOTE}" != ""; then \
         echo "Downloading cache..."; \
-        mkdir -p "$HOME/.cache"; \
         rsync -ra ${RSYNC_REMOTE}/proj/$(uname -m)/ $HOME/.cache/; \
         echo "Finished"; \
         export CC="ccache gcc"; \
@@ -171,7 +175,6 @@ RUN mkdir proj \
         echo "Uploading cache..."; \
         rsync -ra --delete $HOME/.cache/ ${RSYNC_REMOTE}/proj/$(uname -m)/; \
         echo "Finished"; \
-        rm -rf $HOME/.cache; \
         unset CC; \
         unset CXX; \
     fi \
@@ -182,15 +185,15 @@ RUN mkdir proj \
 
 # Build spatialite
 ARG SPATIALITE_VERSION=5.1.0
-RUN if test "${SPATIALITE_VERSION}" != "" -a "$(uname -m)" = "x86_64"; then ( \
+RUN --mount=type=cache,id=alpine-normal-spatialite,target=$HOME/.cache \
+    if test "${SPATIALITE_VERSION}" != "" -a "$(uname -m)" = "x86_64"; then ( \
     curl -LO -fsS https://www.gaia-gis.it/gaia-sins/libspatialite-sources/libspatialite-${SPATIALITE_VERSION}.tar.gz \
     && tar xzf libspatialite-${SPATIALITE_VERSION}.tar.gz \
     && rm -f libspatialite-${SPATIALITE_VERSION}.tar.gz \
     && cd libspatialite-${SPATIALITE_VERSION} \
     && apk add --no-cache minizip-dev \
     && if test "${RSYNC_REMOTE}" != ""; then \
         echo "Downloading cache..."; \
-        mkdir -p "$HOME/.cache"; \
         rsync -ra ${RSYNC_REMOTE}/spatialite/ $HOME/.cache/; \
         echo "Finished"; \
         export CC="ccache gcc"; \
@@ -205,7 +208,6 @@ RUN if test "${SPATIALITE_VERSION}" != "" -a "$(uname -m)" = "x86_64"; then ( \
         echo "Uploading cache..."; \
         rsync -ra --delete $HOME/.cache/ ${RSYNC_REMOTE}/spatialite/; \
         echo "Finished"; \
-        rm -rf $HOME/.cache; \
         unset CC; \
         unset CXX; \
     fi \
@@ -224,7 +226,8 @@ ARG GDAL_RELEASE_DATE
 ARG GDAL_BUILD_IS_RELEASE
 ARG GDAL_REPOSITORY=OSGeo/gdal
 
-RUN if test "${GDAL_VERSION}" = "master"; then \
+RUN --mount=type=cache,id=alpine-normal-gdal,target=$HOME/.cache \
+    if test "${GDAL_VERSION}" = "master"; then \
         export GDAL_VERSION=$(curl -Ls https://api.github.com/repos/${GDAL_REPOSITORY}/commits/HEAD -H "Accept: application/vnd.github.VERSION.sha"); \
         export GDAL_RELEASE_DATE=$(date "+%Y%m%d"); \
     fi \
@@ -241,7 +244,6 @@ RUN if test "${GDAL_VERSION}" = "master"; then \
     && cd gdal \
     && if test "${RSYNC_REMOTE}" != ""; then \
         echo "Downloading cache..."; \
-        mkdir -p "$HOME/.cache"; \
         rsync -ra ${RSYNC_REMOTE}/gdal/$(uname -m)/ $HOME/.cache/; \
         echo "Finished"; \
         # Little trick to avoid issues with Python bindings
@@ -275,7 +277,6 @@ RUN if test "${GDAL_VERSION}" = "master"; then \
         echo "Uploading cache..."; \
         rsync -ra --delete $HOME/.cache/ ${RSYNC_REMOTE}/gdal/$(uname -m)/; \
         echo "Finished"; \
-        rm -rf $HOME/.cache; \
         unset CC; \
         unset CXX; \
     fi \

diff --git a/docker/alpine-small/Dockerfile b/docker/alpine-small/Dockerfile
@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:1
+
 ##
 # osgeo/gdal:alpine-small
 
@@ -11,6 +13,8 @@ FROM alpine:${ALPINE_VERSION} AS builder
 # Derived from osgeo/proj by Howard Butler <[email protected]>
 LABEL maintainer="Even Rouault <[email protected]>"
 
+ENV HOME="/root"
+
 # Setup build env for PROJ
 RUN apk add --no-cache wget curl unzip make libtool autoconf automake pkgconfig g++ sqlite sqlite-dev
 
@@ -56,13 +60,13 @@ ARG RSYNC_REMOTE
 
 # Build PROJ
 ARG PROJ_VERSION=master
-RUN mkdir proj \
+RUN --mount=type=cache,id=alpine-small-proj,target=$HOME/.cache \
+    mkdir proj \
     && curl -L -fsS https://github.com/OSGeo/PROJ/archive/${PROJ_VERSION}.tar.gz \
         | tar xz -C proj --strip-components=1 \
     && cd proj \
     && if test "${RSYNC_REMOTE}" != ""; then \
         echo "Downloading cache..."; \
-        mkdir -p "$HOME/.cache"; \
         rsync -ra ${RSYNC_REMOTE}/proj/$(uname -m)/ $HOME/.cache/; \
         echo "Finished"; \
         export CC="ccache gcc"; \
@@ -86,7 +90,6 @@ RUN mkdir proj \
         echo "Uploading cache..."; \
         rsync -ra --delete $HOME/.cache/ ${RSYNC_REMOTE}/proj/$(uname -m)/; \
         echo "Finished"; \
-        rm -rf $HOME/.cache; \
         unset CC; \
         unset CXX; \
     fi \
@@ -101,7 +104,8 @@ ARG GDAL_RELEASE_DATE
 ARG GDAL_BUILD_IS_RELEASE
 ARG GDAL_REPOSITORY=OSGeo/gdal
 
-RUN if test "${GDAL_VERSION}" = "master"; then \
+RUN --mount=type=cache,id=alpine-small-gdal,target=$HOME/.cache \
+    if test "${GDAL_VERSION}" = "master"; then \
         export GDAL_VERSION=$(curl -Ls https://api.github.com/repos/${GDAL_REPOSITORY}/commits/HEAD -H "Accept: application/vnd.github.VERSION.sha"); \
         export GDAL_RELEASE_DATE=$(date "+%Y%m%d"); \
     fi \
@@ -110,7 +114,6 @@ RUN if test "${GDAL_VERSION}" = "master"; then \
     fi \
     && if test "${RSYNC_REMOTE}" != ""; then \
         echo "Downloading cache..."; \
-        mkdir -p "$HOME/.cache"; \
         rsync -ra ${RSYNC_REMOTE}/gdal/$(uname -m)/ $HOME/.cache/; \
         echo "Finished"; \
         export CC="ccache gcc"; \
@@ -138,7 +141,6 @@ RUN if test "${GDAL_VERSION}" = "master"; then \
         echo "Uploading cache..."; \
         rsync -ra --delete $HOME/.cache/ ${RSYNC_REMOTE}/gdal/$(uname -m)/; \
         echo "Finished"; \
-        rm -rf $HOME/.cache; \
         unset CC; \
         unset CXX; \
     fi \

diff --git a/docker/ubuntu-full/Dockerfile b/docker/ubuntu-full/Dockerfile
@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:1
+
 ##
 # osgeo/gdal:ubuntu-full
 
@@ -13,6 +15,8 @@ FROM $BASE_IMAGE AS builder
 # Derived from osgeo/proj by Howard Butler <[email protected]>
 LABEL maintainer="Even Rouault <[email protected]>"
 
+ENV HOME="/root"
+
 ARG TARGET_ARCH=
 RUN echo ${TARGET_ARCH}
 COPY ./bh-set-envvars.sh /buildscripts/bh-set-envvars.sh
@@ -295,7 +299,8 @@ RUN . /buildscripts/bh-set-envvars.sh \
 
 # Build PROJ
 ARG PROJ_VERSION=master
-RUN . /buildscripts/bh-set-envvars.sh \
+RUN --mount=type=cache,id=ubuntu-full-proj,target=$HOME/.cache \
+    . /buildscripts/bh-set-envvars.sh \
     && /buildscripts/bh-proj.sh
 
 # Build GDAL
@@ -305,7 +310,8 @@ ARG GDAL_BUILD_IS_RELEASE
 ARG GDAL_REPOSITORY=OSGeo/gdal
 
 COPY ./bh-gdal.sh /buildscripts/bh-gdal.sh
-RUN . /buildscripts/bh-set-envvars.sh \
+RUN --mount=type=cache,id=ubuntu-full-gdal,target=$HOME/.cache \
+    . /buildscripts/bh-set-envvars.sh \
     && /buildscripts/bh-gdal.sh
 
 # Build final image

diff --git a/docker/ubuntu-full/bh-gdal.sh b/docker/ubuntu-full/bh-gdal.sh
@@ -23,7 +23,6 @@ curl -L -fsS "https://github.com/${GDAL_REPOSITORY}/archive/${GDAL_VERSION}.tar.
 
     if test "${RSYNC_REMOTE:-}" != ""; then
         echo "Downloading cache..."
-        mkdir -p "$HOME/.cache/"
         rsync -ra "${RSYNC_REMOTE}/gdal/${GCC_ARCH}/" "$HOME/.cache/"
         echo "Finished"
 
@@ -90,7 +89,6 @@ curl -L -fsS "https://github.com/${GDAL_REPOSITORY}/archive/${GDAL_VERSION}.tar.
         rsync -ra --delete "$HOME/.cache/" "${RSYNC_REMOTE}/gdal/${GCC_ARCH}/"
         echo "Finished"
 
-        rm -rf "$HOME/.cache"
         unset CC
         unset CXX
     fi

diff --git a/docker/ubuntu-full/bh-proj.sh b/docker/ubuntu-full/bh-proj.sh
@@ -18,10 +18,10 @@ curl -L -fsS "https://github.com/OSGeo/PROJ/archive/${PROJ_VERSION}.tar.gz" \
 
 (
     cd proj
+    export PROJ_DB_CACHE_PARAM=""
 
     if [ -n "${RSYNC_REMOTE:-}" ]; then
         echo "Downloading cache..."
-        mkdir -p "$HOME/.cache"
         rsync -ra "${RSYNC_REMOTE}/proj/${GCC_ARCH}/" "$HOME/.cache/"
         echo "Finished"
 
@@ -51,7 +51,6 @@ curl -L -fsS "https://github.com/OSGeo/PROJ/archive/${PROJ_VERSION}.tar.gz" \
         rsync -ra --delete "$HOME/.cache/" "${RSYNC_REMOTE}/proj/${GCC_ARCH}/"
         echo "Finished"
 
-        rm -rf "$HOME/.cache"
         unset CC
         unset CXX
     fi

diff --git a/docker/ubuntu-small/Dockerfile b/docker/ubuntu-small/Dockerfile
@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:1
+
 ##
 # osgeo/gdal:ubuntu-small
 
@@ -14,6 +16,8 @@ FROM $BASE_IMAGE AS builder
 # Derived from osgeo/proj by Howard Butler <[email protected]>
 LABEL maintainer="Even Rouault <[email protected]>"
 
+ENV HOME="/root"
+
 ARG TARGET_ARCH=
 RUN echo ${TARGET_ARCH}
 COPY ./bh-set-envvars.sh /buildscripts/bh-set-envvars.sh
@@ -96,14 +100,15 @@ ARG RSYNC_REMOTE
 
 # Build PROJ
 ARG PROJ_VERSION=master
-RUN . /buildscripts/bh-set-envvars.sh \
+RUN --mount=type=cache,id=ubuntu-small-proj,target=$HOME/.cache \
+    . /buildscripts/bh-set-envvars.sh \
     && mkdir proj \
     && curl -L -fsS https://github.com/OSGeo/PROJ/archive/${PROJ_VERSION}.tar.gz \
         | tar xz -C proj --strip-components=1 \
+    && export PROJ_DB_CACHE_PARAM="" \
     && cd proj \
     && if test "${RSYNC_REMOTE:-}" != ""; then \
         echo "Downloading cache..."; \
-        mkdir -p $HOME/.cache; \
         rsync -ra ${RSYNC_REMOTE}/proj/${GCC_ARCH}/ $HOME/.cache/; \
         echo "Finished"; \
         export CC="ccache ${GCC_ARCH}-linux-gnu-gcc"; \
@@ -125,7 +130,6 @@ RUN . /buildscripts/bh-set-envvars.sh \
         echo "Uploading cache..."; \
         rsync -ra --delete $HOME/.cache/ ${RSYNC_REMOTE}/proj/${GCC_ARCH}/; \
         echo "Finished"; \
-        rm -rf $HOME/.cache; \
         unset CC; \
         unset CXX; \
     fi \
@@ -151,7 +155,8 @@ ARG GDAL_RELEASE_DATE
 ARG GDAL_BUILD_IS_RELEASE
 ARG GDAL_REPOSITORY=OSGeo/gdal
 
-RUN . /buildscripts/bh-set-envvars.sh \
+RUN --mount=type=cache,id=ubuntu-small-gdal,target=$HOME/.cache \
+    . /buildscripts/bh-set-envvars.sh \
     && if test "${GDAL_VERSION}" = "master"; then \
         export GDAL_VERSION=$(curl -Ls https://api.github.com/repos/${GDAL_REPOSITORY}/commits/HEAD -H "Accept: application/vnd.github.VERSION.sha"); \
         export GDAL_RELEASE_DATE=$(date "+%Y%m%d"); \
@@ -165,7 +170,6 @@ RUN . /buildscripts/bh-set-envvars.sh \
     && cd gdal \
     && if test "${RSYNC_REMOTE:-}" != ""; then \
         echo "Downloading cache..."; \
-        mkdir -p $HOME/.cache; \
         rsync -ra ${RSYNC_REMOTE}/gdal/${GCC_ARCH}/ $HOME/.cache/; \
         echo "Finished"; \
         # Little trick to avoid issues with Python bindings
@@ -198,7 +202,6 @@ RUN . /buildscripts/bh-set-envvars.sh \
         echo "Uploading cache..."; \
         rsync -ra --delete $HOME/.cache/ ${RSYNC_REMOTE}/gdal/${GCC_ARCH}/; \
         echo "Finished"; \
-        rm -rf $HOME/.cache; \
         unset CC; \
         unset CXX; \
     fi \