Virt-manifest service

.gitignore

@@ -2,6 +2,7 @@ vendor/*
 !vendor/vendor.json
 .idea
 *.iml
+cmd/virt-manifest/virt-manifest
 cmd/virt-controller/virt-controller
 cmd/virt-launcher/virt-launcher
 cmd/virt-handler/virt-handler

cmd/virt-manifest/Dockerfile

@@ -0,0 +1,13 @@
+FROM centos:7
+
+# The jmliger-virt7-upstream-epel-7 has updated libvirt binaries. Since
+# the statically linked libvirt-go code requires a runtime version of libvirt
+# no-lower than what was used when the binary was built.
+RUN yum -y install wget && \
+    wget -P /etc/yum.repos.d https://copr.fedorainfracloud.org/coprs/jmliger/virt7-upstream/repo/epel-7/jmliger-virt7-upstream-epel-7.repo  \
+    && yum -y install libvirt-client \
+    && yum -y clean all
+
+COPY virt-manifest /virt-manifest
+
+ENTRYPOINT [ "/virt-manifest" ]

cmd/virt-manifest/virt-manifest.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/emicklei/go-restful"
+
+	"kubevirt.io/kubevirt/pkg/logging"
+	"kubevirt.io/kubevirt/pkg/virt-handler/virtwrap"
+	"kubevirt.io/kubevirt/pkg/virt-manifest/rest"
+)
+
+func main() {
+	logging.InitializeLogging("virt-manifest")
+	libvirtUri := flag.String("libvirt-uri", "qemu:///system", "Libvirt connection string.")
+	libvirtUser := flag.String("user", "", "Libvirt user")
+	libvirtPasswd := flag.String("pass", "", "Libvirt password")
+	listen := flag.String("listen", "0.0.0.0", "Address where to listen on")
+	port := flag.Int("port", 8186, "Port to listen on")
+	flag.Parse()
+
+	log := logging.DefaultLogger()
+	log.Info().Msg("Starting virt-manifest server")
+
+	log.Info().Msg("Connecting to libvirt")
+
+	domainConn, err := virtwrap.NewConnection(*libvirtUri, *libvirtUser, *libvirtPasswd, 60*time.Second)
+	if err != nil {
+		log.Error().Reason(err).Msg("cannot connect to libvirt")
+		panic(fmt.Sprintf("failed to connect to libvirt: %v", err))
+	}
+	defer domainConn.Close()
+
+	log.Info().Msg("Connected to libvirt")
+
+	ws, err := rest.ManifestService(domainConn)
+	if err != nil {
+		log.Error().Reason(err).Msg("Unable to create REST server.")
+	}
+
+	restful.DefaultContainer.Add(ws)
+	server := &http.Server{Addr: *listen + ":" + strconv.Itoa(*port), Handler: restful.DefaultContainer}
+	log.Info().Msg("Listening for client connections")
+
+	if err := server.ListenAndServe(); err != nil {
+		log.Error().Reason(err).Msg("Unable to start web server.")
+	}
+}

hack/config.sh

@@ -1,4 +1,4 @@
-binaries="cmd/virt-controller cmd/virt-launcher cmd/virt-handler cmd/virt-api cmd/virtctl"
+binaries="cmd/virt-controller cmd/virt-launcher cmd/virt-handler cmd/virt-api cmd/virtctl cmd/virt-manifest"
 docker_images="$binaries images/haproxy images/iscsi-demo-target-tgtd images/vm-killer images/libvirt-kubevirt"
 docker_prefix=kubevirt
 docker_tag=${DOCKER_TAG:-latest}

images/libvirt-kubevirt/Dockerfile

@@ -17,6 +17,9 @@ RUN chmod a+x /usr/local/bin/qemu-x86_64
 COPY kubevirt-sudo /etc/sudoers.d/kubevirt
 RUN chmod 0640 /etc/sudoers.d/kubevirt
 
+COPY libvirtd-limited.sh /libvirtd-limited.sh
+RUN chmod a+x /libvirtd-limited.sh
+
 # libvirtd.sh in this image differs from upstream
 RUN rm -f /libvirtd.sh
 COPY libvirtd.sh /libvirtd.sh

images/libvirt-kubevirt/libvirtd-limited.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/bash
+
+set -xe
+
+/usr/sbin/virtlogd -f /etc/libvirt/virtlogd.conf &
+sleep 5
+
+/usr/sbin/libvirtd

images/libvirt-kubevirt/libvirtd.sh

@@ -63,4 +63,8 @@ fi
 
 echo "cgroup_controllers = [ ]" >> /etc/libvirt/qemu.conf
 
-/usr/sbin/libvirtd -l
+if [[ -n "$LIBVIRTD_DISABLE_TCP" ]]; then
+  /usr/sbin/libvirtd
+else
+  /usr/sbin/libvirtd -l
+fi

manifests/virt-manifest.yaml.in

@@ -0,0 +1,56 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: virt-manifest-service
+spec:
+  ports:
+    - port: 8186
+      targetPort: virt-manifest
+  externalIPs:
+    - "{{ master_ip }}"
+  selector:
+    app: virt-manifest
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: virt-manifest
+spec:
+  template:
+    metadata:
+      labels:
+        app: virt-manifest
+    spec:
+      containers:
+      - name: virt-manifest
+        image: {{ docker_prefix }}/virt-manifest:{{ docker_tag }}
+        imagePullPolicy: IfNotPresent
+        ports:
+          - containerPort: 8186
+            name: "virt-manifest"
+            protocol: "TCP"
+        volumeMounts:
+          - name: libvirt-runtime
+            mountPath: /var/run/libvirt
+        command:
+          - "/virt-manifest"
+          - "--port"
+          - "8186"
+      - name: manifest-libvirtd
+        image: {{ docker_prefix }}/libvirt-kubevirt:{{ docker_tag }}
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          privileged: true
+          runAsUser: 0
+        env:
+        - name: LIBVIRTD_DEFAULT_NETWORK_DEVICE
+          value: eth1
+        volumeMounts:
+          - name: libvirt-runtime
+            mountPath: /var/run/libvirt
+        command: ["/libvirtd-limited.sh"]
+      volumes:
+      - name: libvirt-runtime
+        emptyDir: {}
+      nodeSelector:
+        kubernetes.io/hostname: master

pkg/virt-handler/virtwrap/manager.go

@@ -16,8 +16,10 @@ import (
 	"github.com/jeevatkm/go-model"
 	"github.com/libvirt/libvirt-go"
 	kubev1 "k8s.io/client-go/pkg/api/v1"
+	"k8s.io/client-go/pkg/util/errors"
 	utilwait "k8s.io/client-go/pkg/util/wait"
 	"k8s.io/client-go/tools/record"
+
 	"kubevirt.io/kubevirt/pkg/api/v1"
 	"kubevirt.io/kubevirt/pkg/logging"
 	"kubevirt.io/kubevirt/pkg/virt-handler/virtwrap/api"
@@ -316,16 +318,15 @@ func (l *LibvirtDomainManager) SyncVM(vm *v1.VM) error {
 	var wantedSpec api.DomainSpec
 	mappingErrs := model.Copy(&wantedSpec, vm.Spec.Domain)
 	if len(mappingErrs) > 0 {
-		// TODO: proper aggregation
-		return mappingErrs[0]
+		return errors.NewAggregate(mappingErrs)
 	}
 	dom, err := l.virConn.LookupDomainByName(vm.GetObjectMeta().GetName())
 	if err != nil {
 		// We need the domain but it does not exist, so create it
 		if err.(libvirt.Error).Code == libvirt.ERR_NO_DOMAIN {
 			xmlStr, err := xml.Marshal(&wantedSpec)
 			if err != nil {
-				logging.DefaultLogger().Object(vm).Error().Reason(err).Msg("Generating the domain xmlStr failed.")
+				logging.DefaultLogger().Object(vm).Error().Reason(err).Msg("Generating the domain XML failed.")
 				return err
 			}
 			logging.DefaultLogger().Object(vm).Info().V(3).Msg("Domain XML generated.")

pkg/virt-manifest/defaults.go

@@ -0,0 +1,32 @@
+package virt_manifest
+
+import (
+	"k8s.io/client-go/pkg/util/rand"
+
+	"kubevirt.io/kubevirt/pkg/api/v1"
+)
+
+func AddMinimalVMSpec(vm *v1.VM) {
+	// Make sure the domain name matches the VM name
+	if vm.Spec.Domain == nil {
+		vm.Spec.Domain = new(v1.DomainSpec)
+	}
+	vm.Spec.Domain.Name = vm.ObjectMeta.Name + "-" + rand.String(5)
+
+	AddMinimalDomainSpec(vm.Spec.Domain)
+}
+
+func AddMinimalDomainSpec(dom *v1.DomainSpec) {
+	for idx, graphics := range dom.Devices.Graphics {
+		if graphics.Type == "spice" {
+			if graphics.Listen.Type == "" {
+				dom.Devices.Graphics[idx].Listen.Type = "address"
+			}
+			if ((graphics.Listen.Type == "address") ||
+				(graphics.Listen.Type == "")) &&
+				(graphics.Listen.Address == "") {
+				dom.Devices.Graphics[idx].Listen.Address = "0.0.0.0"
+			}
+		}
+	}
+}

pkg/virt-manifest/mapper.go

@@ -0,0 +1,110 @@
+package virt_manifest
+
+import (
+	"encoding/xml"
+
+	"github.com/jeevatkm/go-model"
+	"github.com/libvirt/libvirt-go"
+	"k8s.io/client-go/pkg/util/errors"
+
+	"kubevirt.io/kubevirt/pkg/api/v1"
+	"kubevirt.io/kubevirt/pkg/logging"
+	"kubevirt.io/kubevirt/pkg/virt-handler/virtwrap"
+	"kubevirt.io/kubevirt/pkg/virt-handler/virtwrap/api"
+)
+
+const (
+	Type_PersistentVolumeClaim = "PersistentVolumeClaim"
+	Type_Network               = "network"
+)
+
+type savedDisk struct {
+	idx  int
+	disk v1.Disk
+}
+
+func ExtractPvc(dom *v1.DomainSpec) (*v1.DomainSpec, []savedDisk) {
+	specCopy := &v1.DomainSpec{}
+	model.Copy(specCopy, dom)
+
+	pvcDisks := []savedDisk{}
+	allDisks := []v1.Disk{}
+
+	for idx, disk := range specCopy.Devices.Disks {
+		if disk.Type == Type_PersistentVolumeClaim {
+			// Save the disk so we can fix it later
+			diskCopy := v1.Disk{}
+			model.Copy(&diskCopy, disk)
+			pvcDisks = append(pvcDisks, savedDisk{disk: diskCopy, idx: idx})
+
+			// Alter the disk record so that libvirt will accept it
+			disk.Type = Type_Network
+			disk.Source.Protocol = "iscsi"
+		}
+		allDisks = append(allDisks, disk)
+	}
+	// Replace the Domain's disks with modified records
+	specCopy.Devices.Disks = allDisks
+
+	return specCopy, pvcDisks
+}
+
+// This is a simplified version of the domain creation portion of SyncVM. This is intended primarily
+// for mapping the VM spec without starting a domain.
+func MapVM(con virtwrap.Connection, vm *v1.VM) (*v1.VM, error) {
+	log := logging.DefaultLogger()
+
+	vmCopy := &v1.VM{}
+	model.Copy(vmCopy, vm)
+
+	specCopy, pvcs := ExtractPvc(vm.Spec.Domain)
+
+	var wantedSpec api.DomainSpec
+	mappingErrs := model.Copy(&wantedSpec, specCopy)
+
+	if len(mappingErrs) > 0 {
+		return nil, errors.NewAggregate(mappingErrs)
+	}
+
+	xmlStr, err := xml.Marshal(&wantedSpec)
+	if err != nil {
+		log.Object(vm).Error().Reason(err).Msg("Generating the domain XML failed.")
+		return nil, err
+	}
+
+	log.Object(vm).Info().V(3).Msg("Domain XML generated.")
+	dom, err := con.DomainDefineXML(string(xmlStr))
+	if err != nil {
+		log.Object(vm).Error().Reason(err).Msg("Defining the VM failed.")
+		return nil, err
+	}
+	log.Object(vm).Info().Msg("Domain defined.")
+
+	defer func() {
+		err = dom.Undefine()
+		if err != nil {
+			log.Object(vm).Warning().Reason(err).Msg("Undefining the domain failed.")
+		} else {
+			log.Object(vm).Info().Msg("Domain defined.")
+		}
+	}()
+
+	domXml, err := dom.GetXMLDesc(libvirt.DOMAIN_XML_MIGRATABLE)
+	if err != nil {
+		log.Object(vm).Error().Reason(err).Msg("Error retrieving domain XML.")
+		return nil, err
+	}
+
+	// api.DomainSpec has xml struct tags.
+	mappedDom := api.DomainSpec{}
+	xml.Unmarshal([]byte(domXml), &mappedDom)
+	model.Copy(vmCopy.Spec.Domain, mappedDom)
+
+	// Re-add the PersistentVolumeClaims that were stripped earlier
+	for _, pvc := range pvcs {
+		vmCopy.Spec.Domain.Devices.Disks[pvc.idx].Type = Type_PersistentVolumeClaim
+		vmCopy.Spec.Domain.Devices.Disks[pvc.idx].Source.Protocol = pvc.disk.Source.Protocol
+	}
+
+	return vmCopy, nil
+}