Create a single entrypoint for the prometheus-operator

Make sure that all kubevirt components which provide logs are selected by the "kubevirt-prometheus-metrics" endpoint. The openshift-operator can look up the registered endpoints behind it and create a scrape config.
webe745 · Sep 24, 2018 · 4ae085e · 4ae085e
1 parent 8aceaaa
commit 4ae085e
Show file tree

Hide file tree

Showing 9 changed files with 141 additions and 44 deletions.
diff --git a/cmd/virt-handler/virt-handler.go b/cmd/virt-handler/virt-handler.go
@@ -25,6 +25,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/golang/glog"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	flag "github.com/spf13/pflag"
 	k8sv1 "k8s.io/api/core/v1"
@@ -35,6 +36,8 @@ import (
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/record"
 
+	"kubevirt.io/kubevirt/pkg/certificates"
+
 	"kubevirt.io/kubevirt/pkg/api/v1"
 	"kubevirt.io/kubevirt/pkg/controller"
 	inotifyinformer "kubevirt.io/kubevirt/pkg/inotify-informer"
@@ -148,11 +151,14 @@ func (app *virtHandlerApp) Run() {
 	// Bootstrapping. From here on the startup order matters
 	stop := make(chan struct{})
 	defer close(stop)
-
+	certStore, err := certificates.GenerateSelfSignedCert("virt-handler", certificates.GetNamespace())
+	if err != nil {
+		glog.Fatalf("unable to generate certificates: %v", err)
+	}
 	go vmController.Run(3, stop)
 
 	http.Handle("/metrics", promhttp.Handler())
-	err = http.ListenAndServe(app.ServiceListen.Address(), nil)
+	err = http.ListenAndServeTLS(app.ServiceListen.Address(), certStore.CurrentPath(), certStore.CurrentPath(), nil)
 	if err != nil {
 		log.Log.Reason(err).Error("Serving prometheus failed.")
 		panic(err)

diff --git a/manifests/dev/prometheus.yaml.in b/manifests/dev/prometheus.yaml.in
@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kubevirt-prometheus-metrics
+  namespace: {{.Namespace}}
+  labels:
+    prometheus.kubevirt.io: ""
+    kubevirt.io: ""
+spec:
+  ports:
+    - name: metrics
+      port: 443
+      targetPort: metrics
+      protocol: TCP
+  selector:
+    prometheus.kubevirt.io: ""
diff --git a/manifests/dev/virt-api.yaml.in b/manifests/dev/virt-api.yaml.in
@@ -35,6 +35,7 @@ spec:
                   ]
       labels:
         kubevirt.io: virt-api
+        prometheus.kubevirt.io: ""
     spec:
       serviceAccountName: kubevirt-apiserver
       containers:
@@ -50,6 +51,9 @@ spec:
             - containerPort: 8443
               name: "virt-api"
               protocol: "TCP"
+            - containerPort: 8443
+              name: "metrics"
+              protocol: "TCP"
           readinessProbe:
             tcpSocket:
               port: 8443

diff --git a/manifests/dev/virt-controller.yaml.in b/manifests/dev/virt-controller.yaml.in
@@ -1,17 +1,3 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: virt-controller
-  namespace: {{.Namespace}}
-  labels:
-    kubevirt.io: "virt-controller"
-spec:
-  ports:
-    - port: 8182
-      targetPort: virt-controller
-  selector:
-    kubevirt.io: virt-controller
----
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
@@ -34,6 +20,7 @@ spec:
                   ]
       labels:
         kubevirt.io: virt-controller
+        prometheus.kubevirt.io: ""
     spec:
       serviceAccountName: kubevirt-controller
       containers:
@@ -45,21 +32,26 @@ spec:
               - "--launcher-image"
               - "{{.DockerPrefix}}/virt-launcher:{{.DockerTag}}"
               - "--port"
-              - "8182"
+              - "8443"
           ports:
-            - containerPort: 8182
+            - containerPort: 8443
               name: "virt-controller"
               protocol: "TCP"
+            - containerPort: 8443
+              name: "metrics"
+              protocol: "TCP"
           livenessProbe:
             failureThreshold: 8
             httpGet:
-              port: 8182
+              scheme: HTTPS
+              port: 8443
               path: /healthz
             initialDelaySeconds: 15
             timeoutSeconds: 10
           readinessProbe:
             httpGet:
-              port: 8182
+              scheme: HTTPS
+              port: 8443
               path: /leader
             initialDelaySeconds: 15
             timeoutSeconds: 10

diff --git a/manifests/dev/virt-handler.yaml.in b/manifests/dev/virt-handler.yaml.in
@@ -22,20 +22,24 @@ spec:
       name: virt-handler
       labels:
         kubevirt.io: virt-handler
+        prometheus.kubevirt.io: ""
     spec:
       serviceAccountName: kubevirt-privileged
       hostPID: true
       containers:
         - name: virt-handler
           ports:
-            - containerPort: 8185
-              hostPort: 8185
+            - containerPort: 8443
+              name: "metrics"
+              protocol: "TCP"
           image: {{.DockerPrefix}}/virt-handler:{{.DockerTag}}
           imagePullPolicy: {{.ImagePullPolicy}}
           command:
             - "virt-handler"
             - "-v"
             - "3"
+            - "--port"
+            - "8443"
             - "--hostname-override"
             - "$(NODE_NAME)"
           securityContext:

diff --git a/manifests/release/kubevirt.yaml.in b/manifests/release/kubevirt.yaml.in
@@ -1,3 +1,21 @@
+# Monitoring
+apiVersion: v1
+kind: Service
+metadata:
+  name: kubevirt-prometheus-metrics
+  namespace: {{.Namespace}}
+  labels:
+    prometheus.kubevirt.io: ""
+    kubevirt.io: ""
+spec:
+  ports:
+    - name: metrics
+      port: 443
+      targetPort: metrics
+      protocol: TCP
+  selector:
+    prometheus.kubevirt.io: ""
+---
 # RBAC
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: ClusterRole
@@ -409,6 +427,7 @@ spec:
                   ]
       labels:
         kubevirt.io: virt-api
+        prometheus.kubevirt.io: ""
     spec:
       serviceAccountName: kubevirt-apiserver
       containers:
@@ -424,6 +443,9 @@ spec:
             - containerPort: 8443
               name: "virt-api"
               protocol: "TCP"
+            - containerPort: 8443
+              name: "metrics"
+              protocol: "TCP"
           readinessProbe:
             tcpSocket:
               port: 8443
@@ -455,6 +477,7 @@ spec:
                   ]
       labels:
         kubevirt.io: virt-controller
+        prometheus.kubevirt.io: ""
     spec:
       serviceAccountName: kubevirt-controller
       containers:
@@ -466,21 +489,23 @@ spec:
               - "--launcher-image"
               - "{{.DockerPrefix}}/virt-launcher:{{.DockerTag}}"
               - "--port"
-              - "8182"
+              - "8443"
           ports:
-            - containerPort: 8182
-              name: "virt-controller"
+            - containerPort: 8443
+              name: "metrics"
               protocol: "TCP"
           livenessProbe:
             failureThreshold: 8
             httpGet:
-              port: 8182
+              scheme: HTTPS
+              port: 8443
               path: /healthz
             initialDelaySeconds: 15
             timeoutSeconds: 10
           readinessProbe:
             httpGet:
-              port: 8182
+              scheme: HTTPS
+              port: 8443
               path: /leader
             initialDelaySeconds: 15
             timeoutSeconds: 10
@@ -512,20 +537,24 @@ spec:
                   ]
       labels:
         kubevirt.io: virt-handler
+        prometheus.kubevirt.io: ""
     spec:
       serviceAccountName: kubevirt-privileged
       hostPID: true
       containers:
         - name: virt-handler
           ports:
-            - containerPort: 8185
-              hostPort: 8185
+            - containerPort: 8443
+              name: "metrics"
+              protocol: "TCP"
           image: {{.DockerPrefix}}/virt-handler:{{.DockerTag}}
           imagePullPolicy: {{.ImagePullPolicy}}
           command:
             - "virt-handler"
             - "-v"
             - "3"
+            -  "--port"
+            - "8443"
             - "--hostname-override"
             - "$(NODE_NAME)"
           securityContext:

diff --git a/pkg/certificates/certificates.go b/pkg/certificates/certificates.go
@@ -0,0 +1,47 @@
+package certificates
+
+import (
+	"io/ioutil"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/util/cert"
+	"k8s.io/client-go/util/cert/triple"
+	"k8s.io/client-go/util/certificate"
+)
+
+func GenerateSelfSignedCert(name string, namespace string) (certificate.FileStore, error) {
+	caKeyPair, _ := triple.NewCA("kubevirt.io")
+	keyPair, _ := triple.NewServerKeyPair(
+		caKeyPair,
+		name+"."+namespace+".pod.cluster.local",
+		name,
+		namespace,
+		"cluster.local",
+		nil,
+		nil,
+	)
+
+	certsDirectory, err := ioutil.TempDir("", "certsdir")
+	if err != nil {
+		return nil, err
+	}
+	store, err := certificate.NewFileStore(name, certsDirectory, certsDirectory, "", "")
+	if err != nil {
+		return nil, err
+	}
+	_, err = store.Update(cert.EncodeCertPEM(keyPair.Cert), cert.EncodePrivateKeyPEM(keyPair.Key))
+	if err != nil {
+		return nil, err
+	}
+	return store, nil
+}
+
+func GetNamespace() string {
+	if data, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil {
+		if ns := strings.TrimSpace(string(data)); len(ns) > 0 {
+			return ns
+		}
+	}
+	return v1.NamespaceSystem
+}
diff --git a/pkg/virt-api/api.go b/pkg/virt-api/api.go
@@ -27,7 +27,6 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"github.com/emicklei/go-restful"
 	"github.com/emicklei/go-restful-openapi"
@@ -45,6 +44,8 @@ import (
 	apiregistrationv1beta1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1"
 	aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
 
+	"kubevirt.io/kubevirt/pkg/certificates"
+
 	"kubevirt.io/kubevirt/pkg/api/v1"
 	"kubevirt.io/kubevirt/pkg/feature-gates"
 	"kubevirt.io/kubevirt/pkg/healthz"
@@ -440,19 +441,10 @@ func (app *virtAPIApp) getClientCert() error {
 	return nil
 }
 
-func getNamespace() string {
-	if data, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil {
-		if ns := strings.TrimSpace(string(data)); len(ns) > 0 {
-			return ns
-		}
-	}
-	return metav1.NamespaceSystem
-}
-
 func (app *virtAPIApp) getSelfSignedCert() error {
 	var ok bool
 
-	namespace := getNamespace()
+	namespace := certificates.GetNamespace()
 	generateCerts := false
 	secret, err := app.virtCli.CoreV1().Secrets(namespace).Get(virtApiCertSecretName, metav1.GetOptions{})
 	if err != nil {
@@ -531,7 +523,7 @@ func (app *virtAPIApp) createWebhook() error {
 }
 
 func (app *virtAPIApp) createValidatingWebhook() error {
-	namespace := getNamespace()
+	namespace := certificates.GetNamespace()
 	registerWebhook := false
 	vmiPathCreate := vmiCreateValidatePath
 	vmiPathUpdate := vmiUpdateValidatePath
@@ -709,7 +701,7 @@ func (app *virtAPIApp) createValidatingWebhook() error {
 }
 
 func (app *virtAPIApp) createMutatingWebhook() error {
-	namespace := getNamespace()
+	namespace := certificates.GetNamespace()
 	registerWebhook := false
 	vmiPath := vmiMutatePath
 
@@ -783,7 +775,7 @@ func (app *virtAPIApp) createMutatingWebhook() error {
 }
 
 func (app *virtAPIApp) createSubresourceApiservice() error {
-	namespace := getNamespace()
+	namespace := certificates.GetNamespace()
 	config, err := kubecli.GetConfig()
 	if err != nil {
 		return err