Implement Docker healthcheck script in Go (minio#7105)

Go script makes it easy to read/maintain. Also updated the timeout
in Dockerfiles from 5s to default 30s and test interval to 1m

Higher timeout makes sense as server may sometimes respond slowly
if under high load as reported in minio#6974

Fixes minio#6974

.gitignore

@@ -22,4 +22,5 @@ parts/
 prime/
 stage/
 .sia_temp/
-config.json
+config.json
+healthcheck

Dockerfile

@@ -11,7 +11,8 @@ RUN  \
      apk add --no-cache git && \
      go get -v -d github.com/minio/minio && \
      cd /go/src/github.com/minio/minio && \
-     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
+     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
+     go build -ldflags "-s -w" -o /usr/bin/healthcheck dockerscripts/healthcheck.go 
 
 FROM alpine:3.7
 
@@ -22,7 +23,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
 EXPOSE 9000
 
 COPY --from=0 /go/bin/minio /usr/bin/minio
-COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
+COPY --from=0 /usr/bin/healthcheck /usr/bin/healthcheck
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 
 RUN  \
      apk add --no-cache ca-certificates 'curl>7.61.0' && \
@@ -32,7 +34,6 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/data"]
 
-HEALTHCHECK --interval=30s --timeout=5s \
-    CMD /usr/bin/healthcheck.sh
+HEALTHCHECK --interval=1m CMD healthcheck
 
 CMD ["minio"]

Dockerfile.dev

@@ -2,7 +2,7 @@ FROM alpine:3.7
 
 LABEL maintainer="Minio Inc <[email protected]>"
 
-COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
+COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck /usr/bin/
 COPY minio /usr/bin/
 
 ENV MINIO_UPDATE off
@@ -14,15 +14,14 @@ RUN \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      chmod +x /usr/bin/minio  && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/healthcheck.sh
+     chmod +x /usr/bin/healthcheck
 
 EXPOSE 9000
 
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/data"]
 
-HEALTHCHECK --interval=30s --timeout=5s \
-    CMD /usr/bin/healthcheck.sh
+HEALTHCHECK --interval=1m CMD healthcheck
 
 CMD ["minio"]

Dockerfile.release

@@ -1,8 +1,22 @@
+FROM golang:1.11.4-alpine3.7
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+
+WORKDIR /go/src/github.com/minio/
+
+RUN  \
+     apk add --no-cache git && \
+     go get -v -d github.com/minio/minio && \
+     cd /go/src/github.com/minio/minio/dockerscripts && \
+     go build -ldflags "-s -w" -o /usr/bin/healthcheck healthcheck.go
+
 FROM alpine:3.7
 
 LABEL maintainer="Minio Inc <[email protected]>"
 
-COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
+COPY --from=0 /usr/bin/healthcheck /usr/bin/healthcheck
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
@@ -14,15 +28,14 @@ RUN \
      curl https://dl.minio.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \
      chmod +x /usr/bin/minio  && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/healthcheck.sh
+     chmod +x /usr/bin/healthcheck
 
 EXPOSE 9000
 
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/data"]
 
-HEALTHCHECK --interval=30s --timeout=5s \
-    CMD /usr/bin/healthcheck.sh
+HEALTHCHECK --interval=1m CMD healthcheck
 
 CMD ["minio"]

Makefile

@@ -66,6 +66,7 @@ coverage: build
 build: checks
 	@echo "Building minio binary to './minio'"
 	@GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
+	@GOFLAGS="" CGO_ENABLED=0 go build --ldflags="-s -w" -o $(PWD)/dockerscripts/healthcheck $(PWD)/dockerscripts/healthcheck.go
 
 docker: build
 	@docker build -t $(TAG) . -f Dockerfile.dev

dockerscripts/healthcheck.go

@@ -0,0 +1,170 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bufio"
+	"crypto/tls"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	xhttp "github.com/minio/minio/cmd/http"
+)
+
+const (
+	initGraceTime = 300
+	healthPath    = "/minio/health/live"
+	timeout       = time.Duration(30 * time.Second)
+	minioProcess  = "minio"
+)
+
+// returns container boot time by finding
+// modtime of /proc/1 directory
+func getStartTime() time.Time {
+	di, err := os.Stat("/proc/1")
+	if err != nil {
+		// Cant stat proc dir successfully, exit with error
+		log.Fatal(err.Error())
+	}
+	return di.ModTime()
+}
+
+// Returns the ip:port of the Minio process
+// running in the container
+func findEndpoint() string {
+	cmd := exec.Command("netstat", "-ntlp")
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		// error getting stdout pipe
+		log.Fatal(err.Error())
+	}
+	if err = cmd.Start(); err != nil {
+		// error starting the command
+		log.Fatal(err.Error())
+	}
+	// split netstat output in rows
+	scanner := bufio.NewScanner(stdout)
+	scanner.Split(bufio.ScanLines)
+	// loop over the rows to find Minio process
+	for scanner.Scan() {
+		if strings.Contains(scanner.Text(), minioProcess) {
+			line := scanner.Text()
+			newLine := strings.Replace(line, ":::", "127.0.0.1:", 1)
+			fields := strings.Fields(newLine)
+			// index 3 in the row has the Local address
+			// find the last index of ":" - address will
+			// have port number after this index
+			i := strings.LastIndex(fields[3], ":")
+			// split address and port
+			addr := fields[3][:i]
+			port := fields[3][i+1:]
+			// add surrounding [] for ip6 address
+			if strings.Count(addr, ":") > 0 {
+				addr = strings.Join([]string{"[", addr, "]"}, "")
+			}
+			// return joint address and port
+			return strings.Join([]string{addr, port}, ":")
+		}
+	}
+	if err = cmd.Wait(); err != nil {
+		// command failed to run
+		log.Fatal(err.Error())
+	}
+	// minio process not found, exit with error
+	os.Exit(1)
+	return ""
+}
+
+func main() {
+	startTime := getStartTime()
+
+	//  In distributed environment like Swarm, traffic is routed
+	//  to a container only when it reports a `healthy` status. So, we exit
+	//  with 0 to ensure healthy status till distributed Minio starts (120s).
+
+	//  Refer: https://github.com/moby/moby/pull/28938#issuecomment-301753272
+
+	if (time.Now().Sub(startTime) / time.Second) < initGraceTime {
+		os.Exit(0)
+	} else {
+		endPoint := findEndpoint()
+		u, err := url.Parse(fmt.Sprintf("http://%s%s", endPoint, healthPath))
+		if err != nil {
+			// Could not parse URL successfully
+			log.Fatal(err.Error())
+		}
+
+		// Minio server may be using self-signed or CA certificates. To avoid
+		// making Docker setup complicated, we skip verifying certificates here.
+		// This is because, following request tests for health status within
+		// containerized environment, i.e. requests are always made to the Minio
+		// server running on the same host.
+		tr := &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		}
+		client := &http.Client{Transport: tr, Timeout: timeout}
+		resp, err := client.Get(u.String())
+		if err != nil {
+			// GET failed exit
+			log.Fatal(err.Error())
+		}
+		if resp.StatusCode == http.StatusOK {
+			// Drain any response.
+			xhttp.DrainBody(resp.Body)
+			// exit with success
+			os.Exit(0)
+		}
+		bodyBytes, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			// Drain any response.
+			xhttp.DrainBody(resp.Body)
+			// GET failed exit
+			log.Fatal(err.Error())
+		}
+		bodyString := string(bodyBytes)
+		// Drain any response.
+		xhttp.DrainBody(resp.Body)
+		// This means sever is configured with https
+		if resp.StatusCode == http.StatusForbidden && bodyString == "SSL required" {
+			// Try with https
+			u.Scheme = "https"
+			resp, err = client.Get(u.String())
+			if err != nil {
+				// GET failed exit
+				log.Fatal(err.Error())
+			}
+			if resp.StatusCode == http.StatusOK {
+				// Drain any response.
+				xhttp.DrainBody(resp.Body)
+				// exit with success
+				os.Exit(0)
+			}
+			// Drain any response.
+			xhttp.DrainBody(resp.Body)
+		}
+	}
+	// Execution reaching here means none of
+	// the success cases were satisfied
+	os.Exit(1)
+}