Fix issue jupyterhub#1793 - Add retryable HTTP client

to image-awaiter so it doesn't bomb on startup if
network is not already available

images/image-awaiter/Dockerfile

@@ -1,15 +1,17 @@
 # compile the code to an executable using an intermediary image
-FROM golang:1.9.2
-
-COPY *.go /go/
-RUN CGO_ENABLED=0 GOOS=linux go build -ldflags '-w -s' -installsuffix cgo -a -o /go/image-awaiter /go/*.go
+FROM golang:1
 
+RUN mkdir -p /build/
+COPY *.mod *.go *.sum /build/
+WORKDIR /build
+RUN go build
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags '-w -s' -installsuffix cgo -a -o out/image-awaiter
 
 
 # present the result within a slimmed image
 FROM scratch
 
-COPY --from=0 /go/image-awaiter /image-awaiter
+COPY --from=0 /build/out/image-awaiter /image-awaiter
 
 
 

images/image-awaiter/daemonset.go

@@ -5,7 +5,10 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"time"
 	"net/http"
+
+	"github.com/hashicorp/go-retryablehttp"
 )
 
 // Partial structure of a Kubernetes DaemonSet object. Note that we want to use
@@ -24,16 +27,22 @@ type DaemonSet struct {
 		DesiredNumberScheduled int `json:"desiredNumberScheduled"`
 		// The number of nodes that should be running the daemon pod and have one
 		// or more of the daemon pod running and ready.
-		NumberReady            int `json:"numberReady"`
+		NumberReady int `json:"numberReady"`
 	} `json:"status"`
 }
 
 // Return a *DaemonSet and the relevant state its in
 func getDaemonSet(transportPtr *http.Transport, server string, headers map[string]string, namespace string, daemonSet string) (*DaemonSet, error) {
-	client := &http.Client{Transport: transportPtr}
+	// Rather than die on startup if the first request fails, retry a few times with backoff before giving up
+	// This can happen if we're e.g. waiting for a service mesh to come up and is generally a good idea.
+	client := retryablehttp.NewClient()
+	client.RetryMax = 5
+	client.RetryWaitMin = time.Duration(5)*time.Second
+	client.HTTPClient.Transport = transportPtr
+
 	url := server + "/apis/apps/v1/namespaces/" + namespace + "/daemonsets/" + daemonSet
 
-	req, err := http.NewRequest("GET", url, nil)
+	req, err := retryablehttp.NewRequest("GET", url, nil)
 	if err != nil {
 		return nil, err
 	}

images/image-awaiter/go.mod

@@ -0,0 +1,5 @@
+module github.com/jupyterhub/zero-to-jupyterhub-k8s/image-awaiter
+
+go 1.15
+
+require github.com/hashicorp/go-retryablehttp v0.6.7

images/image-awaiter/go.sum

@@ -0,0 +1,8 @@
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+github.com/hashicorp/go-retryablehttp v0.6.7 h1:8/CAEZt/+F7kR7GevNHulKkUjLht3CPmn7egmhieNKo=
+github.com/hashicorp/go-retryablehttp v0.6.7/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=