New Docker images (zenml-io#757)

* Update docker files and cloudbuild config * Update reference to docker image for the zenserver * Update default base image name to include python version * Add Dockerfile for local builds
krishkatyal · Jul 13, 2022 · 748df2f · 748df2f
1 parent 29b5079
commit 748df2f
Show file tree

Hide file tree

Showing 10 changed files with 102 additions and 197 deletions.
diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml
@@ -31,5 +31,5 @@ jobs:
         run: |-
           gcloud builds submit \
             --quiet \
-            --config=cloudbuild.yaml \
+            --config=release-cloudbuild.yaml \
             --substitutions=TAG_NAME=${{github.ref_name}} .
diff --git a/cloudbuild-develop.yaml b/cloudbuild-develop.yaml
diff --git a/cloudbuild.yaml b/cloudbuild.yaml
diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile
@@ -1,33 +1,16 @@
-FROM ubuntu:20.04
+ARG PYTHON_VERSION=3.9
+FROM python:${PYTHON_VERSION}-slim AS base
 
-# python
 ENV PYTHONFAULTHANDLER=1 \
     PYTHONUNBUFFERED=1 \
     PYTHONHASHSEED=random \
-    PIP_NO_CACHE_DIR=off \
-    PIP_DISABLE_PIP_VERSION_CHECK=on
-
-RUN apt-get update && \
-  apt-get install --no-install-recommends -q -y \
-  build-essential \
-  ca-certificates \
-  libsnappy-dev \
-  protobuf-compiler \
-  libprotobuf-dev \
-  python3 \
-  python3-dev \
-  python-is-python3 \
-  python3-venv \
-  python3-pip \
-  curl \
-  unzip \
-  git && \
-  apt-get autoclean && \
-  apt-get autoremove --purge
-
-RUN curl -sSL https://bootstrap.pypa.io/get-pip.py | python && \
-  pip install --no-cache-dir --upgrade --pre pip
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1
 
 ARG ZENML_VERSION
+
 # install the given zenml version (default to latest)
-RUN pip install --no-cache-dir zenml${ZENML_VERSION:+==$ZENML_VERSION}
+RUN pip install zenml${ZENML_VERSION:+==$ZENML_VERSION}
+
+FROM base AS server
+RUN pip install zenml${ZENML_VERSION:+==$ZENML_VERSION}[server]
diff --git a/docker/cuda-dev.Dockerfile b/docker/cuda-dev.Dockerfile
diff --git a/docker/cuda.Dockerfile b/docker/cuda.Dockerfile
diff --git a/docker/base-dev.Dockerfile → docker/local-dev.Dockerfile b/docker/base-dev.Dockerfile → docker/local-dev.Dockerfile
@@ -1,8 +1,5 @@
 FROM ubuntu:20.04
 
-WORKDIR /zenml
-
-# python
 ENV PYTHONFAULTHANDLER=1 \
     PYTHONUNBUFFERED=1 \
     PYTHONHASHSEED=random \
@@ -29,23 +26,22 @@ RUN apt-get update && \
   apt-get autoremove --purge
 
 RUN curl -sSL https://install.python-poetry.org | python
-
-# copy project requirement files here to ensure they will be cached.
-COPY pyproject.toml /zenml
+RUN pip install --no-cache-dir --upgrade --pre pip
 
 ENV ZENML_DEBUG=true
 ENV ZENML_ANALYTICS_OPT_IN=false
 ENV VIRTUAL_ENV=/opt/venv
 ENV PATH="$VIRTUAL_ENV/bin:$POETRY_HOME/bin:$PATH"
 RUN python -m venv $VIRTUAL_ENV
 
-RUN pip install --no-cache-dir --upgrade --pre pip
+WORKDIR /zenml
 
-# install dependencies but don't install zenml yet
-# this improves caching as the dependencies don't have to be reinstalled everytime a src file changes
-RUN poetry install --no-root
+# Copy all necessary files for the installation
+COPY README.md pyproject.toml poetry.lock* ./
+# The existence of this __init__.py file allows the poetry install before actually 
+# copying our source files which would invalidate caching
+COPY src/zenml/__init__.py ./src/zenml/
 
-COPY . /zenml
+RUN poetry install
 
-# install zenml
-RUN poetry update && poetry install
+COPY src src
diff --git a/docs/book/collaborate/zenml-server.md b/docs/book/collaborate/zenml-server.md
@@ -148,7 +148,7 @@ mkdir zenserver
 docker run -it -d -p 8080:8000 \
     -v $PWD/zenserver:/zenserver \
     -e ZENML_CONFIG_PATH=/zenserver \
-    zenmldocker/zenml \
+    zenmldocker/zenml-server \
     uvicorn zenml.zen_server.zen_server_api:app --host 0.0.0.0
 ```
 

diff --git a/release-cloudbuild.yaml b/release-cloudbuild.yaml
@@ -0,0 +1,77 @@
+steps:
+  # build base image - python 3.7
+  - name : 'gcr.io/cloud-builders/docker'
+    id: 'build-base'
+    waitFor: ['docker-login']
+    entrypoint: 'bash'
+    args: ['-c', 'docker build --build-arg ZENML_VERSION=$TAG_NAME --build-arg PYTHON_VERSION=3.7 --target base -t $$USERNAME/zenml:$TAG_NAME-py3.7 -t $$USERNAME/zenml:py3.7 -f docker/base.Dockerfile .']
+    secretEnv: ['USERNAME']
+
+  # build base image - python 3.8
+  - name : 'gcr.io/cloud-builders/docker'
+    id: 'build-base'
+    waitFor: ['docker-login']
+    entrypoint: 'bash'
+    args: ['-c', 'docker build --build-arg ZENML_VERSION=$TAG_NAME --build-arg PYTHON_VERSION=3.8 --target base -t $$USERNAME/zenml:$TAG_NAME-py3.8 -t $$USERNAME/zenml:py3.8 -t $$USERNAME/zenml:$TAG_NAME -t $$USERNAME/zenml:latest -f docker/base.Dockerfile .']
+    secretEnv: ['USERNAME']
+
+  # build base image - python 3.9
+  - name : 'gcr.io/cloud-builders/docker'
+    id: 'build-base'
+    waitFor: ['docker-login']
+    entrypoint: 'bash'
+    args: ['-c', 'docker build --build-arg ZENML_VERSION=$TAG_NAME --build-arg PYTHON_VERSION=3.9 --target base -t $$USERNAME/zenml:$TAG_NAME-py3.9 -t $$USERNAME/zenml:py3.9 -f docker/base.Dockerfile .']
+    secretEnv: ['USERNAME']
+
+  # build server image - python 3.7
+  - name : 'gcr.io/cloud-builders/docker'
+    id: 'build-base'
+    waitFor: ['docker-login']
+    entrypoint: 'bash'
+    args: ['-c', 'docker build --build-arg ZENML_VERSION=$TAG_NAME --build-arg PYTHON_VERSION=3.7 -t $$USERNAME/zenml-server:$TAG_NAME-py3.7 -t $$USERNAME/zenml-server:py3.7 -f docker/base.Dockerfile .']
+    secretEnv: ['USERNAME']
+
+  # build server image - python 3.8
+  - name : 'gcr.io/cloud-builders/docker'
+    id: 'build-base'
+    waitFor: ['docker-login']
+    entrypoint: 'bash'
+    args: ['-c', 'docker build --build-arg ZENML_VERSION=$TAG_NAME --build-arg PYTHON_VERSION=3.8 -t $$USERNAME/zenml-server:$TAG_NAME-py3.8 -t $$USERNAME/zenml-server:py3.8 -t $$USERNAME/zenml-server:$TAG_NAME -t $$USERNAME/zenml-server:latest -f docker/base.Dockerfile .']
+    secretEnv: ['USERNAME']
+
+  # build server image - python 3.9
+  - name : 'gcr.io/cloud-builders/docker'
+    id: 'build-base'
+    waitFor: ['docker-login']
+    entrypoint: 'bash'
+    args: ['-c', 'docker build --build-arg ZENML_VERSION=$TAG_NAME --build-arg PYTHON_VERSION=3.9 -t $$USERNAME/zenml-server:$TAG_NAME-py3.9 -t $$USERNAME/zenml-server:py3.9 -f docker/base.Dockerfile .']
+    secretEnv: ['USERNAME']
+
+  # login to dockerhub
+  - name: 'gcr.io/cloud-builders/docker'
+    id: 'docker-login'
+    entrypoint: 'bash'
+    args: ['-c', 'docker login --username=$$USERNAME --password=$$PASSWORD']
+    secretEnv: ['USERNAME', 'PASSWORD']
+
+  # push the base images
+  - name: 'gcr.io/cloud-builders/docker'
+    id: 'push'
+    entrypoint: 'bash'
+    args: [ '-c', 'docker push $$USERNAME/zenml' ]
+    secretEnv: [ 'USERNAME' ]
+
+# push the server images
+- name: 'gcr.io/cloud-builders/docker'
+    id: 'push'
+    entrypoint: 'bash'
+    args: [ '-c', 'docker push $$USERNAME/zenml-server' ]
+    secretEnv: [ 'USERNAME' ]
+
+availableSecrets:
+  secretManager:
+    - versionName: projects/$PROJECT_ID/secrets/docker-password/versions/1
+      env: 'PASSWORD'
+    - versionName: projects/$PROJECT_ID/secrets/docker-username/versions/1
+      env: 'USERNAME'
+timeout: 3600s
diff --git a/src/zenml/utils/docker_utils.py b/src/zenml/utils/docker_utils.py
@@ -15,6 +15,7 @@
 
 import json
 import os
+import sys
 from typing import AbstractSet, Any, Dict, Iterable, List, Optional, cast
 
 import pkg_resources
@@ -29,7 +30,10 @@
 from zenml.utils import string_utils
 from zenml.utils.io_utils import read_file_contents_as_string
 
-DEFAULT_BASE_IMAGE = f"zenmldocker/zenml:{zenml.__version__}"
+DEFAULT_BASE_IMAGE = (
+    f"zenmldocker/zenml:{zenml.__version__}-"
+    f"py{sys.version_info.major}.{sys.version_info.minor}"
+)
 CONTAINER_ZENML_CONFIG_DIR = ".zenconfig"
 
 logger = get_logger(__name__)