Enable multiarch

.circleci/config.yml

@@ -1,29 +1,103 @@
-version: 2
+version: 2.1
+
+docker_build_steps: &docker_build_steps
+  steps:
+    - checkout
+    - run:
+        name: setup test images and run tests
+        command: |
+          export DOCKER_BUILDKIT=1
+          export BUILDKIT_PROGRESS=plain
+          docker context create multi-arch-build
+          docker buildx create multi-arch-build --platform=linux/arm64/v8,linux/amd64 --use
+          docker buildx inspect --bootstrap
+
+          # produce multi-arch image for future publishing
+
+          docker buildx build \
+              --platform=linux/amd64,linux/arm64/v8 \
+              --tag syntestimage:"${DIRNAME}"-regular \
+              --output type=oci,dest=syntestimage.oci \
+              --file ./"${DIRNAME}"/Dockerfile .
+
+          # use build cache to produce single-arch docker images:
+
+          docker buildx build \
+              --platform=linux/amd64 \
+              --tag syntestimage:"${DIRNAME}"-regular \
+              --output type=docker,dest=syntestimage.amd64.tar \
+              --file ./"${DIRNAME}"/Dockerfile .
+
+          docker buildx build \
+              --platform=linux/arm64/v8 \
+              --tag syntestimage:"${DIRNAME}"-regular \
+              --output type=docker,dest=syntestimage.arm64v8.tar \
+              --file ./"${DIRNAME}"/Dockerfile .
+
+    - save_cache:
+        key: syntestimage-{{ .Environment.DIRNAME }}-regular
+        paths:
+          - syntestimage.oci
+          - syntestimage.amd64.tar
+          - syntestimage.arm64v8.tar
 
 docker_test_steps:  &docker_test_steps
   steps:
     - checkout
+    - restore_cache:
+        key: syntestimage-{{ .Environment.DIRNAME }}-regular
     - run:
         name: setup test images and run tests
         command: |
-          docker build -t syntestimage:"$DIRNAME"-regular -f ./"$DIRNAME"/Dockerfile .
-          docker run --rm syntestimage:"$DIRNAME"-regular python -m pip freeze
-          printf "FROM syntestimage:"$DIRNAME"-regular\n\n" > test/Dockerfile_"$DIRNAME"
-          cat test/Dockerfile >> test/Dockerfile_"$DIRNAME"
-          docker build -t syntestimage:"$DIRNAME"-test -f test/Dockerfile_"$DIRNAME" ./test
-          docker run --rm --network host -e "SYN_LOG_LEVEL=DEBUG" syntestimage:"$DIRNAME"-test 2>&1
+          docker load -i syntestimage."${IMAGE_ARCH}".tar
 
-jobs:
+          docker run --rm syntestimage:"${DIRNAME}"-regular python -m pip freeze
+
+          printf "FROM syntestimage:""${DIRNAME}""-regular\n\n" > test/Dockerfile_"${DIRNAME}"
+          cat test/Dockerfile >> test/Dockerfile_"${DIRNAME}"
+          docker build --tag syntestimage:"${DIRNAME}"-test --file test/Dockerfile_"${DIRNAME}" ./test
+          docker run --rm --network host --env "SYN_LOG_LEVEL=DEBUG" syntestimage:"${DIRNAME}"-test 2>&1
 
-  test_311:
+
+executors:
+  amd64:
+    machine:
+      image: ubuntu-2204:current
+
+  arm64:
     machine:
       image: ubuntu-2204:current
+    resource_class: arm.medium
+
+
+jobs:
+  build_311:
+    executor: amd64
+    environment:
+      DIRNAME: python311
+    <<: *docker_build_steps
+
+  test_amd64_311:
+    executor: amd64
+    environment:
+      DIRNAME: python311
+      IMAGE_ARCH: amd64
+    <<: *docker_test_steps
+
+  test_arm64_311:
+    executor: arm64
     environment:
       DIRNAME: python311
+      IMAGE_ARCH: arm64v8
     <<: *docker_test_steps
 
 workflows:
-  version: 2
   run_tests:
     jobs:
-      - test_311
+      - build_311
+      - test_amd64_311:
+          requires:
+          - build_311
+      - test_arm64_311:
+          requires:
+          - build_311

hooks/README.md

@@ -0,0 +1,6 @@
+# Dockerhub Automated build hooks
+
+* `pre_build` sets up multiplatform environment
+* `build` builds *and* pushes the image
+* `push` overrides the step to do nothing
+

hooks/build

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+docker buildx build \
+  --platform=linux/amd64,linux/arm64/v8 \
+  --tag "${IMAGE_NAME}" \
+  --push \
+  --file "${DOCKERFILE_PATH}" \
+  --build-arg BUILD_DATE="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \
+  --build-arg VCS_REF="$(git rev-parse --short HEAD)" \
+  .

hooks/pre_build

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+docker context create multi-arch-build
+docker buildx create --platform=linux/arm64/v8,linux/amd64 --use multi-arch-build
+docker buildx inspect --bootstrap

hooks/push

@@ -0,0 +1 @@
+#!/bin/bash

python311/Dockerfile

@@ -1,60 +1,67 @@
-FROM debian:bookworm-slim
-
-MAINTAINER vEpiphyte <[email protected]>
+FROM python:3.11-bookworm AS base
 
 ENV DEBIAN_FRONTEND="noninteractive"
 ENV PYTHONUNBUFFERED 1
 
-COPY get-pip-23.0.1.py get-pip.py
-COPY pandoc.deb.sha256 pandoc.deb.sha256
-COPY requirements requirements
 COPY python311/rmlist.txt rmlist.txt
+COPY requirements requirements
 # apt get clean / update / upgrade / install required packages / clean up
-# setup locales
 # install python packages
 #  Since we are using --no-deps if a package has a missing dependency here,
 #  it will fail during testing.
+RUN --mount=type=cache,target=/root/.cache \
+    set -ex \
+    && apt-get clean \
+    && apt-get update \
+    && apt-get -y upgrade \
+    && apt-get install --no-install-recommends --no-install-suggests -y build-essential \
+    # Install our packages \
+    && mv requirements/requirements_311.txt requirements/requirements_specific.txt \
+    && pip install --no-deps -r requirements/requirements.txt \
+    # Cleanup \
+    && pip uninstall -y wheel setuptools pip \
+    && while read -r path; do \
+      if [ -e "${path}" ]; then \
+        echo "Removing ${path}" && rm -rf "${path}"; \
+      else \
+        echo "! Path not present: ${path}"; exit 1; \
+      fi \
+    done < rmlist.txt
+
+FROM python:3.11-slim-bookworm
+
+MAINTAINER vEpiphyte <[email protected]>
+
+ENV DEBIAN_FRONTEND="noninteractive"
+ENV PYTHONUNBUFFERED 1
+
+COPY --from=base  /usr/local/lib/python3.11/site-packages/ \
+    /usr/local/lib/python3.11/site-packages/
+
+# apt get clean / update / upgrade / install required packages / clean up
+# setup locales
 # cleanup
-RUN set -ex \
+RUN --mount=type=cache,target=/root/.cache \
+    set -ex \
     && apt-get clean \
     && apt-get update \
     && apt-get -y upgrade \
-    && apt-get install -y locales curl tini nano python3.11 python3.11-distutils \
-    && apt-get install -y build-essential python3.11-dev \
-    # Setup python /pip
-    && update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1 \
-    && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 \
-    && python get-pip.py --break-system-packages \
-    # Install pandoc
-    && curl -L -o pandoc-amd64.deb https://github.com/jgm/pandoc/releases/download/2.14.0.1/pandoc-2.14.0.1-1-amd64.deb \
-    && sha256sum --check pandoc.deb.sha256 \
-    && dpkg -i pandoc-amd64.deb \
-    # Configure locales
+    && apt-get install --no-install-recommends --no-install-suggests -y \
+      locales tini pandoc nano \
+    # Configure locales \
     && echo "en_US.UTF-8 UTF-8" > /etc/locale.gen \
     && locale-gen en_US.UTF-8 \
     && dpkg-reconfigure locales \
     && /usr/sbin/update-locale LANG=en_US.UTF-8 \
-    # Install our packages
-    && mv requirements/requirements_311.txt requirements/requirements_specific.txt \
-    && pip install --no-deps -r requirements/requirements.txt --break-system-packages \
-    # Setup synuser
+    # Setup synuser \
     && groupadd -g 999 synuser \
     && useradd -r --home-dir=/home/synuser -u 999 -g synuser --shell /bin/bash synuser \
     && mkdir -p /home/synuser \
     && chown synuser:synuser /home/synuser \
-    # Cleanup
-    && rm -rf requirements \
-    && apt-get remove -y curl \
-    # Slimmed down version
-    && apt-get remove -y build-essential python3.11-dev \
+    # Slimmed down version \
     && apt-get remove -y --allow-remove-essential e2fsprogs \
     && apt-get autoremove -y \
     && apt-get clean && apt-get purge \
-    && rm -rf /var/lib/apt/lists/* \
-    && rm pandoc.deb.sha256 \
-    && rm pandoc-amd64.deb \
-    && rm get-pip.py \
-    && while read path; do if [ -e $path ]; then echo "Removing ${path}" && rm -rf $path; else echo "! Path not present: ${path}"; exit 1; fi done < rmlist.txt \
-    && rm rmlist.txt
+    && rm -rf /var/lib/apt/lists/*
 
 ENV LANG="en_US.UTF-8" LANGUAGE="en_US.UTF-8" LC_ALL="en_US.UTF-8"

python311/rmlist.txt

@@ -1 +1 @@
-/usr/local/lib/python3.11/dist-packages/tornado/test/test.key
+/usr/local/lib/python3.11/site-packages/tornado/test/test.key

requirements/requirements_dev.txt

@@ -7,4 +7,4 @@ codecov==2.1.13
 pytest==7.3.2
 pytest-cov==4.1.0
 # Support packages for other users
-wheel==0.41.2
+wheel==0.41.2