diff --git a/cmd/virt-handler/virt-handler.go b/cmd/virt-handler/virt-handler.go index 2a4202b22bee..68cc7fd96332 100644 --- a/cmd/virt-handler/virt-handler.go +++ b/cmd/virt-handler/virt-handler.go @@ -25,6 +25,7 @@ import ( "os" "time" + "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus/promhttp" flag "github.com/spf13/pflag" k8sv1 "k8s.io/api/core/v1" @@ -35,6 +36,8 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" + "kubevirt.io/kubevirt/pkg/certificates" + "kubevirt.io/kubevirt/pkg/api/v1" "kubevirt.io/kubevirt/pkg/controller" inotifyinformer "kubevirt.io/kubevirt/pkg/inotify-informer" @@ -148,11 +151,14 @@ func (app *virtHandlerApp) Run() { // Bootstrapping. From here on the startup order matters stop := make(chan struct{}) defer close(stop) - + certStore, err := certificates.GenerateSelfSignedCert("virt-handler", certificates.GetNamespace()) + if err != nil { + glog.Fatalf("unable to generate certificates: %v", err) + } go vmController.Run(3, stop) http.Handle("/metrics", promhttp.Handler()) - err = http.ListenAndServe(app.ServiceListen.Address(), nil) + err = http.ListenAndServeTLS(app.ServiceListen.Address(), certStore.CurrentPath(), certStore.CurrentPath(), nil) if err != nil { log.Log.Reason(err).Error("Serving prometheus failed.") panic(err) diff --git a/manifests/dev/prometheus.yaml.in b/manifests/dev/prometheus.yaml.in new file mode 100644 index 000000000000..50bf969309ca --- /dev/null +++ b/manifests/dev/prometheus.yaml.in @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: kubevirt-prometheus-metrics + namespace: {{.Namespace}} + labels: + prometheus.kubevirt.io: "" + kubevirt.io: "" +spec: + ports: + - name: metrics + port: 443 + targetPort: metrics + protocol: TCP + selector: + prometheus.kubevirt.io: "" diff --git a/manifests/dev/virt-api.yaml.in b/manifests/dev/virt-api.yaml.in index 3bdaaaae74ce..0815a4a1bca1 100644 --- a/manifests/dev/virt-api.yaml.in +++ b/manifests/dev/virt-api.yaml.in @@ -35,6 +35,7 @@ spec: ] labels: kubevirt.io: virt-api + prometheus.kubevirt.io: "" spec: serviceAccountName: kubevirt-apiserver containers: @@ -50,6 +51,9 @@ spec: - containerPort: 8443 name: "virt-api" protocol: "TCP" + - containerPort: 8443 + name: "metrics" + protocol: "TCP" readinessProbe: tcpSocket: port: 8443 diff --git a/manifests/dev/virt-controller.yaml.in b/manifests/dev/virt-controller.yaml.in index 0dd09a31a3a1..35e5f4102579 100644 --- a/manifests/dev/virt-controller.yaml.in +++ b/manifests/dev/virt-controller.yaml.in @@ -1,17 +1,3 @@ -apiVersion: v1 -kind: Service -metadata: - name: virt-controller - namespace: {{.Namespace}} - labels: - kubevirt.io: "virt-controller" -spec: - ports: - - port: 8182 - targetPort: virt-controller - selector: - kubevirt.io: virt-controller ---- apiVersion: extensions/v1beta1 kind: Deployment metadata: @@ -34,6 +20,7 @@ spec: ] labels: kubevirt.io: virt-controller + prometheus.kubevirt.io: "" spec: serviceAccountName: kubevirt-controller containers: @@ -45,21 +32,26 @@ spec: - "--launcher-image" - "{{.DockerPrefix}}/virt-launcher:{{.DockerTag}}" - "--port" - - "8182" + - "8443" ports: - - containerPort: 8182 + - containerPort: 8443 name: "virt-controller" protocol: "TCP" + - containerPort: 8443 + name: "metrics" + protocol: "TCP" livenessProbe: failureThreshold: 8 httpGet: - port: 8182 + scheme: HTTPS + port: 8443 path: /healthz initialDelaySeconds: 15 timeoutSeconds: 10 readinessProbe: httpGet: - port: 8182 + scheme: HTTPS + port: 8443 path: /leader initialDelaySeconds: 15 timeoutSeconds: 10 diff --git a/manifests/dev/virt-handler.yaml.in b/manifests/dev/virt-handler.yaml.in index 34d596c70538..34ef73af3e47 100644 --- a/manifests/dev/virt-handler.yaml.in +++ b/manifests/dev/virt-handler.yaml.in @@ -22,20 +22,24 @@ spec: name: virt-handler labels: kubevirt.io: virt-handler + prometheus.kubevirt.io: "" spec: serviceAccountName: kubevirt-privileged hostPID: true containers: - name: virt-handler ports: - - containerPort: 8185 - hostPort: 8185 + - containerPort: 8443 + name: "metrics" + protocol: "TCP" image: {{.DockerPrefix}}/virt-handler:{{.DockerTag}} imagePullPolicy: {{.ImagePullPolicy}} command: - "virt-handler" - "-v" - "3" + - "--port" + - "8443" - "--hostname-override" - "$(NODE_NAME)" securityContext: diff --git a/manifests/release/kubevirt.yaml.in b/manifests/release/kubevirt.yaml.in index 234bdce2a8fc..b728937244aa 100644 --- a/manifests/release/kubevirt.yaml.in +++ b/manifests/release/kubevirt.yaml.in @@ -1,3 +1,21 @@ +# Monitoring +apiVersion: v1 +kind: Service +metadata: + name: kubevirt-prometheus-metrics + namespace: {{.Namespace}} + labels: + prometheus.kubevirt.io: "" + kubevirt.io: "" +spec: + ports: + - name: metrics + port: 443 + targetPort: metrics + protocol: TCP + selector: + prometheus.kubevirt.io: "" +--- # RBAC apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole @@ -409,6 +427,7 @@ spec: ] labels: kubevirt.io: virt-api + prometheus.kubevirt.io: "" spec: serviceAccountName: kubevirt-apiserver containers: @@ -424,6 +443,9 @@ spec: - containerPort: 8443 name: "virt-api" protocol: "TCP" + - containerPort: 8443 + name: "metrics" + protocol: "TCP" readinessProbe: tcpSocket: port: 8443 @@ -455,6 +477,7 @@ spec: ] labels: kubevirt.io: virt-controller + prometheus.kubevirt.io: "" spec: serviceAccountName: kubevirt-controller containers: @@ -466,21 +489,23 @@ spec: - "--launcher-image" - "{{.DockerPrefix}}/virt-launcher:{{.DockerTag}}" - "--port" - - "8182" + - "8443" ports: - - containerPort: 8182 - name: "virt-controller" + - containerPort: 8443 + name: "metrics" protocol: "TCP" livenessProbe: failureThreshold: 8 httpGet: - port: 8182 + scheme: HTTPS + port: 8443 path: /healthz initialDelaySeconds: 15 timeoutSeconds: 10 readinessProbe: httpGet: - port: 8182 + scheme: HTTPS + port: 8443 path: /leader initialDelaySeconds: 15 timeoutSeconds: 10 @@ -512,20 +537,24 @@ spec: ] labels: kubevirt.io: virt-handler + prometheus.kubevirt.io: "" spec: serviceAccountName: kubevirt-privileged hostPID: true containers: - name: virt-handler ports: - - containerPort: 8185 - hostPort: 8185 + - containerPort: 8443 + name: "metrics" + protocol: "TCP" image: {{.DockerPrefix}}/virt-handler:{{.DockerTag}} imagePullPolicy: {{.ImagePullPolicy}} command: - "virt-handler" - "-v" - "3" + - "--port" + - "8443" - "--hostname-override" - "$(NODE_NAME)" securityContext: diff --git a/pkg/certificates/certificates.go b/pkg/certificates/certificates.go new file mode 100644 index 000000000000..a244fe558bc2 --- /dev/null +++ b/pkg/certificates/certificates.go @@ -0,0 +1,47 @@ +package certificates + +import ( + "io/ioutil" + "strings" + + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/cert" + "k8s.io/client-go/util/cert/triple" + "k8s.io/client-go/util/certificate" +) + +func GenerateSelfSignedCert(name string, namespace string) (certificate.FileStore, error) { + caKeyPair, _ := triple.NewCA("kubevirt.io") + keyPair, _ := triple.NewServerKeyPair( + caKeyPair, + name+"."+namespace+".pod.cluster.local", + name, + namespace, + "cluster.local", + nil, + nil, + ) + + certsDirectory, err := ioutil.TempDir("", "certsdir") + if err != nil { + return nil, err + } + store, err := certificate.NewFileStore(name, certsDirectory, certsDirectory, "", "") + if err != nil { + return nil, err + } + _, err = store.Update(cert.EncodeCertPEM(keyPair.Cert), cert.EncodePrivateKeyPEM(keyPair.Key)) + if err != nil { + return nil, err + } + return store, nil +} + +func GetNamespace() string { + if data, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil { + if ns := strings.TrimSpace(string(data)); len(ns) > 0 { + return ns + } + } + return v1.NamespaceSystem +} diff --git a/pkg/virt-api/api.go b/pkg/virt-api/api.go index 4a5e551d8c8b..b1d9e47d3d17 100644 --- a/pkg/virt-api/api.go +++ b/pkg/virt-api/api.go @@ -27,7 +27,6 @@ import ( "net/http" "os" "path/filepath" - "strings" "github.com/emicklei/go-restful" "github.com/emicklei/go-restful-openapi" @@ -45,6 +44,8 @@ import ( apiregistrationv1beta1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1" aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + "kubevirt.io/kubevirt/pkg/certificates" + "kubevirt.io/kubevirt/pkg/api/v1" "kubevirt.io/kubevirt/pkg/feature-gates" "kubevirt.io/kubevirt/pkg/healthz" @@ -440,19 +441,10 @@ func (app *virtAPIApp) getClientCert() error { return nil } -func getNamespace() string { - if data, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil { - if ns := strings.TrimSpace(string(data)); len(ns) > 0 { - return ns - } - } - return metav1.NamespaceSystem -} - func (app *virtAPIApp) getSelfSignedCert() error { var ok bool - namespace := getNamespace() + namespace := certificates.GetNamespace() generateCerts := false secret, err := app.virtCli.CoreV1().Secrets(namespace).Get(virtApiCertSecretName, metav1.GetOptions{}) if err != nil { @@ -531,7 +523,7 @@ func (app *virtAPIApp) createWebhook() error { } func (app *virtAPIApp) createValidatingWebhook() error { - namespace := getNamespace() + namespace := certificates.GetNamespace() registerWebhook := false vmiPathCreate := vmiCreateValidatePath vmiPathUpdate := vmiUpdateValidatePath @@ -709,7 +701,7 @@ func (app *virtAPIApp) createValidatingWebhook() error { } func (app *virtAPIApp) createMutatingWebhook() error { - namespace := getNamespace() + namespace := certificates.GetNamespace() registerWebhook := false vmiPath := vmiMutatePath @@ -783,7 +775,7 @@ func (app *virtAPIApp) createMutatingWebhook() error { } func (app *virtAPIApp) createSubresourceApiservice() error { - namespace := getNamespace() + namespace := certificates.GetNamespace() config, err := kubecli.GetConfig() if err != nil { return err diff --git a/pkg/virt-controller/watch/application.go b/pkg/virt-controller/watch/application.go index 6fe74c26836c..3087f9d3927e 100644 --- a/pkg/virt-controller/watch/application.go +++ b/pkg/virt-controller/watch/application.go @@ -28,6 +28,7 @@ import ( "time" "github.com/emicklei/go-restful" + "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus/promhttp" flag "github.com/spf13/pflag" k8sv1 "k8s.io/api/core/v1" @@ -39,6 +40,8 @@ import ( "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/client-go/tools/record" + "kubevirt.io/kubevirt/pkg/certificates" + "kubevirt.io/kubevirt/pkg/controller" featuregates "kubevirt.io/kubevirt/pkg/feature-gates" "kubevirt.io/kubevirt/pkg/kubecli" @@ -170,13 +173,17 @@ func Execute() { func (vca *VirtControllerApp) Run() { logger := log.Log + certStore, err := certificates.GenerateSelfSignedCert("virt-controller", certificates.GetNamespace()) + if err != nil { + glog.Fatalf("unable to generate certificates: %v", err) + } stop := make(chan struct{}) defer close(stop) go func() { httpLogger := logger.With("service", "http") httpLogger.Level(log.INFO).Log("action", "listening", "interface", vca.BindAddress, "port", vca.Port) http.Handle("/metrics", promhttp.Handler()) - if err := http.ListenAndServe(vca.Address(), nil); err != nil { + if err := http.ListenAndServeTLS(vca.Address(), certStore.CurrentPath(), certStore.CurrentPath(), nil); err != nil { golog.Fatal(err) } }()