Switch to decentralized libvirtd-per-pod

Signed-off-by: David Vossel <[email protected]>

cmd/registry-disk-v1alpha/entry-point.sh

@@ -33,6 +33,7 @@ fi
 IMAGE_PATH=/disk/$IMAGE_NAME
 IMAGE_EXTENSION=$(echo $IMAGE_NAME | sed  -n -e 's/^.*\.\(.*\)$/\1/p')
 
+mkdir -p $COPY_PATH
 echo $IMAGE_NAME | grep -q -e "raw" -e "qcow2"
 if [ $? -ne 0 ]; then
 	IMAGE_EXTENSION="raw"
@@ -50,8 +51,6 @@ else
 fi
 echo "copied $IMAGE_PATH to $COPY_PATH.${IMAGE_EXTENSION}"
 
-trap "rm -f ${COPY_PATH}.${IMAGE_EXTENSION}" TERM INT HUP QUIT EXIT ERR
-
 touch /tmp/healthy
 while [ -f "${COPY_PATH}.${IMAGE_EXTENSION}" ]; do
 	sleep 5

cmd/virt-handler/Dockerfile

@@ -20,13 +20,6 @@ FROM fedora:27
 
 MAINTAINER "The KubeVirt Project" <[email protected]>
 
-ENV LIBVIRT_VERSION 3.7.0
-
-RUN dnf -y install libvirt-client-${LIBVIRT_VERSION} genisoimage && \
-    groupadd --gid 107 qemu && \
-    useradd --uid 107 --gid 107 qemu && \
-    dnf -y clean all
-
 COPY virt-handler /virt-handler
 
 ENTRYPOINT [ "/virt-handler" ]

cmd/virt-handler/virt-handler.go

@@ -21,13 +21,10 @@ package main
 
 import (
 	"fmt"
-	"net/http"
 	"os"
 
 	"time"
 
-	"github.com/emicklei/go-restful"
-	"github.com/libvirt/libvirt-go"
 	flag "github.com/spf13/pflag"
 	k8sv1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/fields"
@@ -38,50 +35,36 @@ import (
 	"k8s.io/client-go/tools/record"
 
 	"kubevirt.io/kubevirt/pkg/api/v1"
-	cloudinit "kubevirt.io/kubevirt/pkg/cloud-init"
-	configdisk "kubevirt.io/kubevirt/pkg/config-disk"
 	"kubevirt.io/kubevirt/pkg/controller"
 	inotifyinformer "kubevirt.io/kubevirt/pkg/inotify-informer"
 	"kubevirt.io/kubevirt/pkg/kubecli"
 	"kubevirt.io/kubevirt/pkg/log"
-	registrydisk "kubevirt.io/kubevirt/pkg/registry-disk"
 	"kubevirt.io/kubevirt/pkg/service"
 	"kubevirt.io/kubevirt/pkg/virt-handler"
-	"kubevirt.io/kubevirt/pkg/virt-handler/rest"
-	"kubevirt.io/kubevirt/pkg/virt-handler/virtwrap"
-	virt_api "kubevirt.io/kubevirt/pkg/virt-handler/virtwrap/api"
-	virtcache "kubevirt.io/kubevirt/pkg/virt-handler/virtwrap/cache"
-	virtcli "kubevirt.io/kubevirt/pkg/virt-handler/virtwrap/cli"
-	"kubevirt.io/kubevirt/pkg/virt-handler/virtwrap/isolation"
 	virtlauncher "kubevirt.io/kubevirt/pkg/virt-launcher"
+	virt_api "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
+	virtcache "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/cache"
 	watchdog "kubevirt.io/kubevirt/pkg/watchdog"
 )
 
 const (
-	defaultWatchdogTimeout = 30 * time.Second
+	defaultWatchdogTimeout = 15 * time.Second
 
 	// Default port that virt-handler listens on.
 	defaultPort = 8185
 
 	// Default address that virt-handler listens on.
 	defaultHost = "0.0.0.0"
 
-	// The URI connection string supplied to libvirt. By default, we connect to system-mode daemon of QEMU.
-	libvirtUri = "qemu:///system"
-
 	hostOverride = ""
 
 	virtShareDir = "/var/run/kubevirt"
-
-	ephemeralDiskDir = "/var/run/libvirt/kubevirt-ephemeral-disk"
 )
 
 type virtHandlerApp struct {
 	service.ServiceListen
-	service.ServiceLibvirt
 	HostOverride            string
 	VirtShareDir            string
-	EphemeralDiskDir        string
 	WatchdogTimeoutDuration time.Duration
 }
 
@@ -100,29 +83,6 @@ func (app *virtHandlerApp) Run() {
 	logger := log.Log
 	logger.V(1).Level(log.INFO).Log("hostname", app.HostOverride)
 
-	err := cloudinit.SetLocalDirectory(app.EphemeralDiskDir + "/cloud-init-data")
-	if err != nil {
-		panic(err)
-	}
-	err = registrydisk.SetLocalDirectory(app.EphemeralDiskDir + "/registry-disk-data")
-	if err != nil {
-		panic(err)
-	}
-
-	go func() {
-		for {
-			if res := libvirt.EventRunDefaultImpl(); res != nil {
-				// Report the error somehow or break the loop.
-				logger.Reason(res).Error("Listening to libvirt events failed.")
-			}
-		}
-	}()
-	domainConn, err := virtcli.NewConnection(app.LibvirtUri, "", "", 10*time.Second)
-	if err != nil {
-		panic(fmt.Sprintf("failed to connect to libvirtd: %v", err))
-	}
-	defer domainConn.Close()
-
 	// Create event recorder
 	virtCli, err := kubecli.GetKubevirtClient()
 	if err != nil {
@@ -133,10 +93,6 @@ func (app *virtHandlerApp) Run() {
 	// TODO what is scheme used for in Recorder?
 	recorder := broadcaster.NewRecorder(scheme.Scheme, k8sv1.EventSource{Component: "virt-handler", Host: app.HostOverride})
 
-	domainManager, err := virtwrap.NewLibvirtDomainManager(domainConn,
-		recorder,
-		isolation.NewSocketBasedIsolationDetector(app.VirtShareDir),
-	)
 	if err != nil {
 		panic(err)
 	}
@@ -146,12 +102,10 @@ func (app *virtHandlerApp) Run() {
 		panic(err)
 	}
 
-	configDiskClient := configdisk.NewConfigDiskClient(virtCli)
-
 	// Wire VM controller
 
 	// Wire Domain controller
-	domainSharedInformer, err := virtcache.NewSharedInformer(domainConn)
+	domainSharedInformer, err := virtcache.NewSharedInformer(app.VirtShareDir)
 	if err != nil {
 		panic(err)
 	}
@@ -181,11 +135,9 @@ func (app *virtHandlerApp) Run() {
 		cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
 
 	vmController := virthandler.NewController(
-		domainManager,
 		recorder,
 		virtCli,
 		app.HostOverride,
-		configDiskClient,
 		app.VirtShareDir,
 		int(app.WatchdogTimeoutDuration.Seconds()),
 		vmSharedInformer,
@@ -198,43 +150,28 @@ func (app *virtHandlerApp) Run() {
 	stop := make(chan struct{})
 	defer close(stop)
 
-	go vmController.Run(3, stop)
-
-	// TODO add a http handler which provides health check
-
-	migrationHostInfo := rest.NewMigrationHostInfo(isolation.NewSocketBasedIsolationDetector(app.VirtShareDir))
-	ws := new(restful.WebService)
-	ws.Route(ws.GET("/api/v1/namespaces/{namespace}/virtualmachines/{name}/migrationHostInfo").To(migrationHostInfo.MigrationHostInfo))
-	restful.DefaultContainer.Add(ws)
-	server := &http.Server{Addr: app.Address(), Handler: restful.DefaultContainer}
-	server.ListenAndServe()
+	vmController.Run(3, stop)
 }
 
 func (app *virtHandlerApp) AddFlags() {
 	app.InitFlags()
 
 	app.BindAddress = defaultHost
 	app.Port = defaultPort
-	app.LibvirtUri = libvirtUri
 
 	app.AddCommonFlags()
-	app.AddLibvirtFlags()
 
 	flag.StringVar(&app.HostOverride, "hostname-override", hostOverride,
 		"Name under which the node is registered in kubernetes, where this virt-handler instance is running on")
 
 	flag.StringVar(&app.VirtShareDir, "kubevirt-share-dir", virtShareDir,
 		"Shared directory between virt-handler and virt-launcher")
 
-	flag.StringVar(&app.EphemeralDiskDir, "ephemeral-disk-dir", ephemeralDiskDir,
-		"Base directory for ephemeral disk data")
-
 	flag.DurationVar(&app.WatchdogTimeoutDuration, "watchdog-timeout", defaultWatchdogTimeout,
 		"Watchdog file timeout")
 }
 
 func main() {
-	libvirt.EventRegisterDefaultImpl()
 	app := &virtHandlerApp{}
 	service.Setup(app)
 	log.InitializeLogging("virt-handler")

cmd/virt-launcher/Dockerfile

@@ -16,16 +16,30 @@
 # Copyright 2017 Red Hat, Inc.
 #
 
-FROM fedora:27
+FROM kubevirt/libvirt
 
 MAINTAINER "The KubeVirt Project" <[email protected]>
 
-RUN dnf -y install socat && \
-    groupadd --gid 107 qemu && \
-    useradd --uid 107 --gid 107 qemu && \
-    dnf -y clean all
+RUN dnf -y install \
+  socat \
+  genisoimage \
+  util-linux \
+  libcgroup-tools \
+  ethtool \
+  sudo && dnf -y clean all && \
+  test $(id -u qemu) = 107 # make sure that the qemu user really is 107
 
 COPY sock-connector /sock-connector
 COPY virt-launcher /virt-launcher
 
-ENTRYPOINT [ "/virt-launcher" ]
+COPY kubevirt-sudo /etc/sudoers.d/kubevirt
+RUN chmod 0640 /etc/sudoers.d/kubevirt
+
+# libvirtd.sh in this image differs from upstream
+RUN rm -f /libvirtd.sh
+COPY libvirtd.sh /libvirtd.sh
+RUN chmod a+x /libvirtd.sh
+
+COPY entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT [ "/entrypoint.sh" ]

cmd/virt-launcher/entrypoint.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+./virt-launcher $@

images/libvirt-kubevirt/libvirtd.sh → cmd/virt-launcher/libvirtd.sh

@@ -21,7 +21,7 @@ set -xe
 
 # HACK
 # Use hosts's /dev to see new devices and allow macvtap
-mkdir /dev.container && {
+mkdir -p /dev.container && {
     mount --rbind /dev /dev.container
     mount --rbind /host-dev /dev
 
@@ -36,27 +36,27 @@ mkdir /dev.container && {
     [[ -e /dev.container/kvm ]] && keep kvm
 }
 
-mkdir /sys.net.container && {
+mkdir -p /sys.net.container && {
     mount --rbind /sys/class/net /sys.net.container
     mount --rbind /host-sys/class/net /sys/class/net
 }
 
-mkdir /sys.devices.container && {
+mkdir -p /sys.devices.container && {
     mount --rbind /sys/devices /sys.devices.container
     mount --rbind /host-sys/devices /sys/devices
 }
 
 # If no cpuacct,cpu is present, symlink it to cpu,cpuacct
 # Otherwise libvirt and our emulator get confused
-if [ ! -d "/host-sys/fs/cgroup/cpuacct,cpu" ]; then
-    echo "Creating cpuacct,cpu cgroup symlink"
-    mount -o remount,rw /host-sys/fs/cgroup
-    cd /host-sys/fs/cgroup
-    ln -s cpu,cpuacct cpuacct,cpu
-    mount -o remount,ro /host-sys/fs/cgroup
-fi
+#if [ ! -d "/host-sys/fs/cgroup/cpuacct,cpu" ]; then
+#    echo "Creating cpuacct,cpu cgroup symlink"
+#    mount -o remount,rw /host-sys/fs/cgroup
+#    cd /host-sys/fs/cgroup
+#    ln -s cpu,cpuacct cpuacct,cpu
+#    mount -o remount,ro /host-sys/fs/cgroup
+#fi
 
-mount --rbind /host-sys/fs/cgroup /sys/fs/cgroup
+#mount --rbind /host-sys/fs/cgroup /sys/fs/cgroup
 
 mkdir -p /var/log/kubevirt
 touch /var/log/kubevirt/qemu-kube.log
@@ -82,13 +82,9 @@ if [[ -n "$LIBVIRTD_DEFAULT_NETWORK_DEVICE" ]]; then
   </forward>
 </network>
 EOX
-    ln -s /etc/libvirt/qemu/networks/default.xml /etc/libvirt/qemu/networks/autostart/default.xml
+    ln -s -f /etc/libvirt/qemu/networks/default.xml /etc/libvirt/qemu/networks/autostart/default.xml
 fi
 
 echo "cgroup_controllers = [ ]" >>/etc/libvirt/qemu.conf
 
-if [[ -n "$LIBVIRTD_DISABLE_TCP" ]]; then
-    /usr/sbin/libvirtd
-else
-    /usr/sbin/libvirtd -l
-fi
+/usr/sbin/libvirtd