Merge pull request Kaggle#492 from Kaggle/allow-parallel-branch-build-ci

Allow parallel branch builds
songjiaru · Feb 26, 2019 · ed0dea4 · ed0dea4
2 parents 7780a6f + d6ca2b0
commit ed0dea4
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 10 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,7 @@
+ARG BASE_TAG=5.2.0
+
 FROM gcr.io/kaggle-images/python-tensorflow-whl:1.12.0-py36 as tensorflow_whl
-FROM continuumio/anaconda3:5.2.0
+FROM continuumio/anaconda3:${BASE_TAG}
 
 ADD clean-layer.sh  /tmp/clean-layer.sh
 ADD patches/nbconvert-extensions.tpl /opt/kaggle/nbconvert-extensions.tpl

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -15,6 +15,8 @@ pipeline {
     GIT_COMMIT_AUTHOR = sh(returnStdout: true, script:"git log --format='%an' -n 1 HEAD").trim()
     GIT_COMMIT_SUMMARY = "`<https://github.com/Kaggle/docker-python/commit/${GIT_COMMIT}|${GIT_COMMIT_SHORT}>` ${GIT_COMMIT_SUBJECT} - ${GIT_COMMIT_AUTHOR}"
     SLACK_CHANNEL = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"master\" ]]; then echo \"#kernelops\"; else echo \"#builds\"; fi").trim()
+    PRETEST_TAG = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"master\" ]]; then echo \"ci-pretest\"; else echo \"${GIT_BRANCH}-pretest\"; fi").trim()
+    STAGING_TAG = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"master\" ]]; then echo \"staging\"; else echo \"${GIT_BRANCH}-staging\"; fi").trim()
   }
 
   stages {
@@ -34,7 +36,7 @@ pipeline {
           set -exo pipefail
 
           date
-          ./push ci-pretest
+          ./push ${PRETEST_TAG}
         '''
       }
     }
@@ -56,7 +58,7 @@ pipeline {
           set -exo pipefail
 
           date
-          ./push staging
+          ./push ${STAGING_TAG}
         '''
       }
     }
@@ -73,7 +75,7 @@ pipeline {
         sh '''#!/bin/bash
           set -exo pipefail
           docker image prune -f # remove previously built image to prevent disk from filling up
-          ./build --gpu | ts
+          ./build --gpu --base-image-tag ${STAGING_TAG} | ts
         '''
       }
     }
@@ -85,7 +87,7 @@ pipeline {
           set -exo pipefail
 
           date
-          ./push --gpu ci-pretest
+          ./push --gpu ${PRETEST_TAG}
         '''
       }
     }
@@ -109,7 +111,7 @@ pipeline {
           set -exo pipefail
 
           date
-          ./push --gpu staging
+          ./push --gpu ${STAGING_TAG}
         '''
       }
     }

diff --git a/build b/build
@@ -7,14 +7,16 @@ Usage: $0 [OPTIONS]
 Build a new Python Docker image.
 
 Options:
-    -g, --gpu       Build an image with GPU support.
-    -c, --use-cache Use layer cache when building a new image.
+    -g, --gpu                 Build an image with GPU support.
+    -c, --use-cache           Use layer cache when building a new image.
+    -b, --base-image-tag TAG  Base image tag. Defaults to value defined in DOCKERFILE.
 EOF
 }
 
 CACHE_FLAG='--no-cache'
 DOCKERFILE='Dockerfile'
 IMAGE_TAG='kaggle/python-build'
+BUILD_ARGS=''
 
 while :; do
     case "$1" in 
@@ -29,6 +31,15 @@ while :; do
         -c|--use-cache)
             CACHE_FLAG=''
             ;;
+        -b|--base-image-tag)
+            if [[ -z $2 ]]; then
+                usage
+                printf 'ERROR: No TAG specified after the %s flag.\n' "$1" >&2
+                exit
+            fi
+            BUILD_ARGS="--build-arg BASE_TAG=$2"
+            shift # skip the flag value
+            ;;
         -?*)
             usage
             printf 'ERROR: Unknown option: %s\n' "$1" >&2
@@ -44,6 +55,7 @@ done
 readonly CACHE_FLAG
 readonly DOCKERFILE
 readonly IMAGE_TAG
+readonly BUILD_ARGS
 
 set -x
-docker build --rm --pull $CACHE_FLAG -t "$IMAGE_TAG" -f "$DOCKERFILE" .
+docker build --rm --pull $CACHE_FLAG -t "$IMAGE_TAG" -f "$DOCKERFILE" $BUILD_ARGS .
diff --git a/gpu.Dockerfile b/gpu.Dockerfile
@@ -1,6 +1,8 @@
+ARG BASE_TAG=staging
+
 FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu16.04 AS nvidia
 FROM gcr.io/kaggle-images/python-tensorflow-whl:1.12.0-py36 as tensorflow_whl
-FROM gcr.io/kaggle-images/python:staging
+FROM gcr.io/kaggle-images/python:${BASE_TAG}
 
 ADD clean-layer.sh  /tmp/clean-layer.sh