utilize k8s-api for daemonset status instead of listing pods

chartpress.yaml

@@ -13,7 +13,7 @@ charts:
       network-tools:
         valuesPath: singleuser.networkTools.image
       image-awaiter:
-        valuesPath: imagePuller.hook.image
+        valuesPath: prePuller.hook.image
       singleuser-sample:
         valuesPath: singleuser.image
         buildArgs:

images/image-awaiter/Dockerfile

@@ -13,10 +13,14 @@ COPY --from=0 /go/image-awaiter /image-awaiter
 
 
 
-# To debug this image locally with some cluster...
+# To debug / develop this code
+# ----------------------------
+# 1. Setup a kubectl proxy
 # > kubectl proxy --port=8080
-# > docker build --tag <name:tag> .
-# > docker run -it --rm --net=host <name:tag>
 
-ENTRYPOINT [ "/image-awaiter" ]
-CMD [ "-debug" ]
+# 2. Try the API using the proxy...
+# > curl http://localhost:8080/apis/apps/v1beta2/namespaces/<namespace>/demonsets/hook-image-puller
+
+# 3. Try the container using the proxy...
+# > docker build --tag <name:tag> .
+# > docker run -it --rm --net=host <name:tag> /image-awaiter -debug -namespace <namespace> -daemonset hook-image-puller

images/image-awaiter/README.md

@@ -1,22 +1,24 @@
-# Image Image-Awaiter
+# Image-Awaiter
 
-## What is the image image-awaiter?
+## What is the image-awaiter?
 
-The image awaiter is designed to await a set of user-specified images to become
-present on all nodes that may have user pods assigned to them. It is a small go
-program that repeatedly checks if the given list of images exists on all 
-scheduleable nodes and then exits.
+The image awaiter works in conjunction with a daemonset that will schedule pods
+that pull images to all nodes. It works by repeatedly checking the daemonset's
+pods are ready, and exits when they are.
 
 ## Why would one use it?
 
-This is used as a [helm hook](https://github.com/kubernetes/helm/blob/master/docs/charts_hooks.md)
-to wait for user images to be present on all nodes before we restart the
-hub. This cuts down the amount of time it takes for a user server to start,
-since the image no longer needs to be pulled. For large images this can cut down
-startup time from almost ten minutes to a few seconds.
+Because it can delay the hub to be upgraded before the relevant images are made
+available, and that can for large images cut down startup time from almost ten 
+minutes to a few seconds.
 
 ## FAQ
 
+### What technical knowledge is needed to understand this?
+
+You need to know about [Kubernetes Jobs](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/) and [Kubernetes DaemonSets](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/), about [Helm and helm hooks](https://github.com/kubernetes/helm/blob/master/docs/charts_hooks.md), 
+and about the programming language Go.
+
 ### Why is this project in Go? Isn't the Jupyter Infrastructure ecosystem mostly Python?
 
 The size of the image needed to run this image-awaiter needs to be as small as possible in order

images/image-awaiter/daemonset.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+)
+
+// Partial structure of a Kubernetes DaemonSet object
+// Only contains fields we will actively be using, to make JSON parsing nicer
+type DaemonSet struct {
+	Kind   string `json:"kind"`
+	Status struct {
+		DesiredNumberScheduled int `json:"desiredNumberScheduled"`
+		NumberReady            int `json:"numberReady"`
+	} `json:"status"`
+}
+
+// Return a *DaemonSet and the relevant state its in
+func getDaemonSet(transportPtr *http.Transport, server string, headers map[string]string, namespace string, daemonSet string) (*DaemonSet, error) {
+	client := &http.Client{Transport: transportPtr}
+	url := server + "/apis/apps/v1beta2/namespaces/" + namespace + "/daemonsets/" + daemonSet
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	for k, v := range headers {
+		req.Header.Add(k, v)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	ds := &DaemonSet{}
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	err = json.Unmarshal(data, &ds)
+
+	if ds.Kind != "DaemonSet" {
+		// Something went wrong!
+		return nil, fmt.Errorf(fmt.Sprintf("Can not parse API response as DaemonSet: %s", string(data)))
+	}
+
+	return ds, err
+}
+
+func isImagesPresent(ds *DaemonSet) bool {
+	desired := ds.Status.DesiredNumberScheduled
+	ready := ds.Status.NumberReady
+
+	log.Printf("%d of %d nodes currently has the required images.", desired, ready)
+
+	return desired == ready
+}

images/image-awaiter/main.go

@@ -1,36 +1,12 @@
-// This program will be run as a helm hook before an actual helm upgrade have
-// started. It will simply wait for image pulling to complete by the
-// helm-image-puller daemonset's pods, it will poll these pods and exit when
-// they are all running.
-
-// TODO:
-// - Consider what the query param called 'includeUninitialized' will do
-// - Consider unschedulable nodes and how the DS will schedule pods on them
-//   make sure the daemonsets schedules the pods wisely.
+// Program used to delay a helm upgrade process until all relevant nodes have
+// pulled required images. It is an image-awaiter. It can simply wait because
+// the hook-image-puller daemonset that will get the images pulled is already
+// started when this job starts. When all images are pulled, this job exits.
 
 /*
-FUTURE REWORK:
-Stop using /api/v1/pods and instead use /api/v1/namespaces/<ns>/daemonsets/hook-image-puller/status
-- Current solution: curl http://localhost:8080/api/v1/pods?labelSelector=component=hook-image-puller
-- K8s 1.8 solution: curl http://localhost:8080/apis/apps/v1beta2/namespaces/<ns>/demonsets/hook-image-puller/status
-- K8s 1.9 solution: curl http://localhost:8080/api/v1/namespaces/<ns>/demonsets/hook-image-puller/status
-
-{
-	"kind": "DaemonSet",
-	"apiVersion": "apps/v1beta2",
-
-	...
-
-	"status": {
-		"currentNumberScheduled": 2,
-		"numberMisscheduled": 0,
-		"desiredNumberScheduled": 2,
-		"numberReady": 2,
-		"observedGeneration": 1,
-		"updatedNumberScheduled": 2,
-		"numberAvailable": 2
-	}
-}
+K8s API options - currently using 1.8
+- K8s 1.8 API: curl http://localhost:8080/apis/apps/v1beta2/namespaces/<ns>/demonsets/<ds>
+- K8s 1.9 API: curl http://localhost:8080/apis/apps/v1/namespaces/<ns>/demonsets/<ds>
 */
 
 package main
@@ -100,14 +76,17 @@ func makeHeaders(debug bool, authTokenPath string) (map[string]string, error) {
 }
 
 func main() {
-	var caPath, clientCertPath, clientKeyPath, authTokenPath, apiServerAddress string
-	var debug bool
+	var caPath, clientCertPath, clientKeyPath, authTokenPath, apiServerAddress, namespace, daemonSet string
 	flag.StringVar(&caPath, "ca-path", "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", "Path to CA bundle used to verify kubernetes master")
 	flag.StringVar(&clientCertPath, "client-certificate-path", "", "Path to client certificate used to authenticate with kubernetes server")
 	flag.StringVar(&clientKeyPath, "client-key-path", "", "Path to client certificate key used to authenticate with kubernetes server")
 	flag.StringVar(&authTokenPath, "auth-token-path", "/var/run/secrets/kubernetes.io/serviceaccount/token", "Auth Token to use when making API requests")
 	flag.StringVar(&apiServerAddress, "api-server-address", "", "Address of the Kubernetes API Server to contact")
+	flag.StringVar(&namespace, "namespace", "", "Namespace of the DaemonSet that will perform image pulling")
+	flag.StringVar(&daemonSet, "daemonset", "hook-image-puller", "The name DaemonSet that will perform image pulling")
+	var debug bool
 	flag.BoolVar(&debug, "debug", false, "Communicate through a 'kubectl proxy --port 8080' setup instead.")
+
 	flag.Parse()
 
 	if debug {
@@ -125,12 +104,12 @@ func main() {
 	}
 
 	for {
-		pods, err := getImagePullerPods(transportPtr, apiServerAddress, headers)
+		ds, err := getDaemonSet(transportPtr, apiServerAddress, headers, namespace, daemonSet)
 		if err != nil {
 			log.Fatal(err)
 		}
 
-		if isImagesPresent(pods) {
+		if isImagesPresent(ds) {
 			log.Printf("All images present on all nodes!")
 			break
 		}

jupyterhub/templates/image-puller/_helper.yaml

@@ -1,7 +1,7 @@
 {{ define "jupyterhub.imagePuller.daemonset" }}
 # template that returns a image-puller daemonset, used for
-# - pre-upgrade image pulling before helm makes an upgrade (auto-deleted on completion)
-# - continuous image pulling for newly added nodes
+# - pre-helm-upgrade image pulling (pulls images before helm upgrades, temporary)
+# - continuous image pulling (pulls images for cluster autoscalers, persistent)
 apiVersion: extensions/v1beta1
 kind: DaemonSet
 metadata:
@@ -10,9 +10,9 @@ metadata:
     {{ if .hook }}
     # create it before installs/upgrades ...
     "helm.sh/hook": pre-install,pre-upgrade
-    # ... before the image-awaiter job ...
+    # ... and before the image-awaiter job ...
     "helm.sh/hook-weight": "-10"
-    # ... and delete it after
+    # ... but then delete it
     "helm.sh/hook-delete-policy": hook-succeeded,hook-failed
     {{ end }}
   labels:
@@ -54,7 +54,7 @@ spec:
           - -c
           - echo "Pulling complete"
       {{ end }}
-      {{ range $k, $v := .top.Values.imagePuller.extraImages }}
+      {{ range $k, $v := .top.Values.prePuller.extraImages }}
       - name: image-pull-{{ $k }}
         image: {{ $v.name }}:{{ $v.tag }}
         imagePullPolicy: IfNotPresent
@@ -65,5 +65,5 @@ spec:
       {{ end }}
       containers:
       - name: pause
-        image: {{ .top.Values.imagePuller.pause.image.name }}:{{ .top.Values.imagePuller.pause.image.tag }}
+        image: {{ .top.Values.prePuller.pause.image.name }}:{{ .top.Values.prePuller.pause.image.tag }}
 {{- end }}

jupyterhub/templates/image-puller/daemonset.yaml

@@ -1,9 +1,9 @@
 # will delete itself unless the helm upgrade process is aborted
-{{ if .Values.imagePuller.hook.enabled }}
+{{ if .Values.prePuller.hook.enabled }}
 {{ template "jupyterhub.imagePuller.daemonset" (dict "hook" true "name" "hook-image-puller" "top" .) }}
 {{- end }}
 ---
 # will remain running and pull images on new nodes added to the cluster
-{{ if .Values.imagePuller.continuous.enabled }}
+{{ if .Values.prePuller.continuous.enabled }}
 {{ template "jupyterhub.imagePuller.daemonset" (dict "hook" false "name" "continuous-image-puller" "top" .) }}
 {{- end }}

jupyterhub/templates/image-puller/job.yaml

@@ -3,7 +3,7 @@
 # its' pods running. If all those pods are running they must have pulled all the
 # required images on all nodes as they are used as init containers with a dummy
 # command.
-{{ if .Values.imagePuller.hook.enabled }}
+{{ if .Values.prePuller.hook.enabled }}
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -20,27 +20,14 @@ spec:
       serviceAccountName: image-awaiter
       {{- end }}
       containers:
-        - image: {{ .Values.imagePuller.hook.image.name }}:{{ .Values.imagePuller.hook.image.tag }}
+        - image: {{ .Values.prePuller.hook.image.name }}:{{ .Values.prePuller.hook.image.tag }}
           name: image-awaiter
           imagePullPolicy: IfNotPresent
           command:
             - /image-awaiter
             - -ca-path=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
             - -auth-token-path=/var/run/secrets/kubernetes.io/serviceaccount/token
             - -api-server-address=https://$(KUBERNETES_SERVICE_HOST):$(KUBERNETES_SERVICE_PORT)
-              # This is explicitly *not* quoted, since we are passing
-              # This to a go program without any shell interpolation
-            - {{ .Values.singleuser.image.name }}:{{ .Values.singleuser.image.tag }}
-
-              {{ if not .Values.singleuser.cloudMetadata.enabled }}
-            - {{ .Values.singleuser.networkTools.image.name }}:{{ .Values.singleuser.networkTools.image.tag }}
-              {{ end }}
-              {{ range $key, $value := .Values.imagePuller.extraImages }}
-            - {{ $value.name }}:{{ $value.tag }}
-              {{ end }}
-          env:
-            - name: NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
+            - -namespace={{ .Release.Namespace }}
+            - -daemonset=hook-image-puller
 {{ end }}