remove all references to gracefulshutdown and watchdog files from vir…

…t-launcher

Signed-off-by: David Vossel <[email protected]>

cmd/virt-handler/BUILD.bazel

@@ -19,11 +19,12 @@ go_library(
         "//pkg/virt-config:go_default_library",
         "//pkg/virt-handler:go_default_library",
         "//pkg/virt-handler/cache:go_default_library",
+        "//pkg/virt-handler/cmd-client:go_default_library",
         "//pkg/virt-handler/isolation:go_default_library",
         "//pkg/virt-handler/rest:go_default_library",
         "//pkg/virt-handler/selinux:go_default_library",
-        "//pkg/virt-launcher:go_default_library",
         "//pkg/virt-launcher/virtwrap/api:go_default_library",
+        "//pkg/watchdog:go_default_library",
         "//staging/src/github.com/golang/glog:go_default_library",
         "//staging/src/kubevirt.io/client-go/api/v1:go_default_library",
         "//staging/src/kubevirt.io/client-go/kubecli:go_default_library",

cmd/virt-handler/virt-handler.go

@@ -57,11 +57,12 @@ import (
 	virtconfig "kubevirt.io/kubevirt/pkg/virt-config"
 	virthandler "kubevirt.io/kubevirt/pkg/virt-handler"
 	virtcache "kubevirt.io/kubevirt/pkg/virt-handler/cache"
+	cmdclient "kubevirt.io/kubevirt/pkg/virt-handler/cmd-client"
 	"kubevirt.io/kubevirt/pkg/virt-handler/isolation"
 	"kubevirt.io/kubevirt/pkg/virt-handler/rest"
 	"kubevirt.io/kubevirt/pkg/virt-handler/selinux"
-	virtlauncher "kubevirt.io/kubevirt/pkg/virt-launcher"
 	virt_api "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
+	"kubevirt.io/kubevirt/pkg/watchdog"
 )
 
 const (
@@ -184,7 +185,22 @@ func (app *virtHandlerApp) Run() {
 		panic(err)
 	}
 
-	virtlauncher.InitializeSharedDirectories(app.VirtShareDir)
+	// Legacy directory for watchdog files
+	err = os.MkdirAll(watchdog.WatchdogFileDirectory(app.VirtShareDir), 0755)
+	if err != nil {
+		panic(err)
+	}
+
+	// Legacy Directory for graceful shutdown trigger files.
+	err = os.MkdirAll(filepath.Join(app.VirtShareDir, "graceful-shutdown-trigger"), 0755)
+	if err != nil {
+		panic(err)
+	}
+
+	err = os.MkdirAll(cmdclient.SocketsDirectory(app.VirtShareDir), 0755)
+	if err != nil {
+		panic(err)
+	}
 
 	app.namespace, err = clientutil.GetNamespace()
 	if err != nil {
@@ -201,9 +217,10 @@ func (app *virtHandlerApp) Run() {
 		glog.Fatalf("Error constructing migration tls config: %v", err)
 	}
 
+	// Legacy support, Remove this informer once we no longer support
+	// VMIs with graceful shutdown trigger
 	gracefulShutdownInformer := cache.NewSharedIndexInformer(
-		inotifyinformer.NewFileListWatchFromClient(
-			virtlauncher.GracefulShutdownTriggerDir(app.VirtShareDir)),
+		inotifyinformer.NewFileListWatchFromClient(filepath.Join(app.VirtShareDir, "graceful-shutdown-trigger")),
 		&virt_api.Domain{},
 		0,
 		cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})

cmd/virt-launcher/BUILD.bazel

@@ -21,7 +21,6 @@ go_library(
         "//pkg/virt-launcher/virtwrap/cli:go_default_library",
         "//pkg/virt-launcher/virtwrap/cmd-server:go_default_library",
         "//pkg/virt-launcher/virtwrap/util:go_default_library",
-        "//pkg/watchdog:go_default_library",
         "//staging/src/kubevirt.io/client-go/api/v1:go_default_library",
         "//staging/src/kubevirt.io/client-go/log:go_default_library",
         "//vendor/github.com/libvirt/libvirt-go:go_default_library",

cmd/virt-launcher/virt-launcher.go

@@ -33,6 +33,7 @@ import (
 	"github.com/spf13/pflag"
 
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
 	utilwait "k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/watch"
 
@@ -53,11 +54,9 @@ import (
 	virtcli "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/cli"
 	cmdserver "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/cmd-server"
 	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/util"
-	"kubevirt.io/kubevirt/pkg/watchdog"
 )
 
 const defaultStartTimeout = 3 * time.Minute
-const defaultWatchdogInterval = 5 * time.Second
 
 func init() {
 	// must registry the event impl before doing anything else.
@@ -146,34 +145,6 @@ func startDomainEventMonitoring(
 	}
 }
 
-func startWatchdogTicker(watchdogFile string, watchdogInterval time.Duration, stopChan chan struct{}, uid string) (done chan struct{}) {
-	err := watchdog.WatchdogFileUpdate(watchdogFile, uid)
-	if err != nil {
-		panic(err)
-	}
-
-	log.Log.Infof("Watchdog file created at %s", watchdogFile)
-	done = make(chan struct{})
-
-	go func() {
-		defer close(done)
-
-		ticker := time.NewTicker(watchdogInterval).C
-		for {
-			select {
-			case <-stopChan:
-				return
-			case <-ticker:
-				err := watchdog.WatchdogFileUpdate(watchdogFile, uid)
-				if err != nil {
-					panic(err)
-				}
-			}
-		}
-	}()
-	return done
-}
-
 func initializeDirs(virtShareDir string,
 	ephemeralDiskDir string,
 	containerDiskDir string,
@@ -183,12 +154,8 @@ func initializeDirs(virtShareDir string,
 	mask := syscall.Umask(0)
 	defer syscall.Umask(mask)
 
-	err := virtlauncher.InitializeSharedDirectories(virtShareDir)
-	if err != nil {
-		panic(err)
-	}
+	err := virtlauncher.InitializePrivateDirectories(filepath.Join("/var/run/kubevirt-private", uid))
 
-	err = virtlauncher.InitializePrivateDirectories(filepath.Join("/var/run/kubevirt-private", uid))
 	if err != nil {
 		panic(err)
 	}
@@ -265,7 +232,7 @@ func waitForFinalNotify(deleteNotificationSent chan watch.Event,
 	// First attempt to wait for domain event to occur as a part of the normal shutdown flow.
 	// If that fails, call Kill on the domain and wait for the event again.
 	// If that that fails, exit. We did our best to shutdown the domain gracefully. We can't block
-	// the pod forever. Virt-handler will learn of the domain's exit through the watchdog file expire.
+	// the pod forever. Virt-handler will learn of the domain's exit through monitoring cmd server socket.
 
 	killTimeout := time.After(15 * time.Second)
 	timedOut := false
@@ -325,7 +292,6 @@ func main() {
 	name := pflag.String("name", "", "Name of the VirtualMachineInstance")
 	uid := pflag.String("uid", "", "UID of the VirtualMachineInstance")
 	namespace := pflag.String("namespace", "", "Namespace of the VirtualMachineInstance")
-	watchdogInterval := pflag.Duration("watchdog-update-interval", defaultWatchdogInterval, "Interval at which watchdog file should be updated")
 	readinessFile := pflag.String("readiness-file", "/var/run/kubevirt-infra/healthy", "Pod looks for this file to determine when virt-launcher is initialized")
 	gracePeriodSeconds := pflag.Int("grace-period-seconds", 30, "Grace period to observe before sending SIGTERM to vm process")
 	useEmulation := pflag.Bool("use-emulation", false, "Use software emulation")
@@ -368,11 +334,6 @@ func main() {
 	// Start libvirtd, virtlogd, and establish libvirt connection
 	stopChan := make(chan struct{})
 
-	watchdogFile := watchdog.WatchdogFileFromNamespaceName(*virtShareDir,
-		*namespace,
-		*name)
-	watchdogDone := startWatchdogTicker(watchdogFile, *watchdogInterval, stopChan, *uid)
-
 	err = util.SetupLibvirt()
 	if err != nil {
 		panic(err)
@@ -416,20 +377,22 @@ func main() {
 		panic(err)
 	}
 
-	gracefulShutdownTriggerFile := virtlauncher.GracefulShutdownTriggerFromNamespaceName(*virtShareDir,
-		*namespace,
-		*name)
-	err = virtlauncher.GracefulShutdownTriggerClear(gracefulShutdownTriggerFile)
-	if err != nil {
-		log.Log.Reason(err).Errorf("Error clearing shutdown trigger file %s.", gracefulShutdownTriggerFile)
-		panic(err)
-	}
-
 	gracefulShutdownCallback := func() {
-		err := domainManager.MarkGracefulShutdownVMI(vm)
+		err := wait.PollImmediate(time.Second, 15*time.Second, func() (bool, error) {
+			err := domainManager.MarkGracefulShutdownVMI(vm)
+			if err != nil {
+				log.Log.Reason(err).Errorf("Unable to signal graceful shutdown")
+				return false, err
+			}
+
+			return true, nil
+		})
+
 		if err != nil {
-			log.Log.Reason(err).Errorf("Unable to signal graceful shutdown")
+			log.Log.Reason(err).Errorf("Gave up attempting to signal graceful shutdown")
 		}
+
+		log.Log.Object(vm).Info("Successfully signaled graceful shutdown")
 	}
 
 	finalShutdownCallback := func(pid int) {
@@ -467,7 +430,6 @@ func main() {
 	domain := waitForDomainUUID(*qemuTimeout, events, signalStopChan, domainManager)
 	if domain != nil {
 		mon := virtlauncher.NewProcessMonitor(domain.Spec.UUID,
-			gracefulShutdownTriggerFile,
 			*gracePeriodSeconds,
 			finalShutdownCallback,
 			gracefulShutdownCallback)
@@ -484,7 +446,6 @@ func main() {
 
 	close(stopChan)
 	<-cmdServerDone
-	<-watchdogDone
 
 	log.Log.Info("Exiting...")
 }

pkg/virt-handler/BUILD.bazel

@@ -7,6 +7,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/controller:go_default_library",
+        "//pkg/ephemeral-disk-utils:go_default_library",
         "//pkg/handler-launcher-com/cmd/v1:go_default_library",
         "//pkg/host-disk:go_default_library",
         "//pkg/util:go_default_library",
@@ -18,7 +19,6 @@ go_library(
         "//pkg/virt-handler/device-manager:go_default_library",
         "//pkg/virt-handler/isolation:go_default_library",
         "//pkg/virt-handler/migration-proxy:go_default_library",
-        "//pkg/virt-launcher:go_default_library",
         "//pkg/virt-launcher/virtwrap/api:go_default_library",
         "//pkg/virt-launcher/virtwrap/network:go_default_library",
         "//pkg/watchdog:go_default_library",
@@ -51,7 +51,6 @@ go_test(
         "//pkg/virt-config:go_default_library",
         "//pkg/virt-handler/cmd-client:go_default_library",
         "//pkg/virt-handler/isolation:go_default_library",
-        "//pkg/virt-launcher:go_default_library",
         "//pkg/virt-launcher/virtwrap/api:go_default_library",
         "//pkg/virt-launcher/virtwrap/network:go_default_library",
         "//pkg/watchdog:go_default_library",

pkg/virt-handler/vm.go

@@ -48,6 +48,7 @@ import (
 	"kubevirt.io/client-go/kubecli"
 	"kubevirt.io/client-go/log"
 	"kubevirt.io/kubevirt/pkg/controller"
+	diskutils "kubevirt.io/kubevirt/pkg/ephemeral-disk-utils"
 	cmdv1 "kubevirt.io/kubevirt/pkg/handler-launcher-com/cmd/v1"
 	hostdisk "kubevirt.io/kubevirt/pkg/host-disk"
 	virtutil "kubevirt.io/kubevirt/pkg/util"
@@ -58,7 +59,6 @@ import (
 	device_manager "kubevirt.io/kubevirt/pkg/virt-handler/device-manager"
 	"kubevirt.io/kubevirt/pkg/virt-handler/isolation"
 	migrationproxy "kubevirt.io/kubevirt/pkg/virt-handler/migration-proxy"
-	virtlauncher "kubevirt.io/kubevirt/pkg/virt-launcher"
 	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
 	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/network"
 	"kubevirt.io/kubevirt/pkg/watchdog"
@@ -842,6 +842,27 @@ func (d *VirtualMachineController) migrationTargetExecute(key string,
 	return nil
 }
 
+// Legacy, remove once we're certain we are no longer supporting
+// VMIs running with the old graceful shutdown trigger logic
+func gracefulShutdownTriggerFromNamespaceName(baseDir string, namespace string, name string) string {
+	triggerFile := namespace + "_" + name
+	return filepath.Join(baseDir, "graceful-shutdown-trigger", triggerFile)
+}
+
+// Legacy, remove once we're certain we are no longer supporting
+// VMIs running with the old graceful shutdown trigger logic
+func vmGracefulShutdownTriggerClear(baseDir string, vmi *v1.VirtualMachineInstance) error {
+	triggerFile := gracefulShutdownTriggerFromNamespaceName(baseDir, vmi.Namespace, vmi.Name)
+	return diskutils.RemoveFile(triggerFile)
+}
+
+// Legacy, remove once we're certain we are no longer supporting
+// VMIs running with the old graceful shutdown trigger logic
+func vmHasGracefulShutdownTrigger(baseDir string, vmi *v1.VirtualMachineInstance) (bool, error) {
+	triggerFile := gracefulShutdownTriggerFromNamespaceName(baseDir, vmi.Namespace, vmi.Name)
+	return diskutils.FileExists(triggerFile)
+}
+
 // Determine if gracefulShutdown has been triggered by virt-launcher
 func (d *VirtualMachineController) hasGracefulShutdownTrigger(vmi *v1.VirtualMachineInstance, domain *api.Domain) (bool, error) {
 
@@ -855,7 +876,7 @@ func (d *VirtualMachineController) hasGracefulShutdownTrigger(vmi *v1.VirtualMac
 
 	// Fallback to detecting the old way of reporting gracefulshutdown, via file.
 	// We keep this around in order to ensure backwards compatibility
-	return virtlauncher.VmHasGracefulShutdownTrigger(d.virtShareDir, vmi)
+	return vmHasGracefulShutdownTrigger(d.virtShareDir, vmi)
 }
 
 func (d *VirtualMachineController) defaultExecute(key string,
@@ -1116,7 +1137,7 @@ func (d *VirtualMachineController) processVmCleanup(vmi *v1.VirtualMachineInstan
 
 	// If the VMI is using the old graceful shutdown trigger on
 	// a hostmount, make sure to clear that file still.
-	err := virtlauncher.VmGracefulShutdownTriggerClear(d.virtShareDir, vmi)
+	err := vmGracefulShutdownTriggerClear(d.virtShareDir, vmi)
 	if err != nil {
 		return err
 	}

pkg/virt-handler/vm_test.go

@@ -54,7 +54,6 @@ import (
 	"kubevirt.io/kubevirt/pkg/testutils"
 	cmdclient "kubevirt.io/kubevirt/pkg/virt-handler/cmd-client"
 	"kubevirt.io/kubevirt/pkg/virt-handler/isolation"
-	virtlauncher "kubevirt.io/kubevirt/pkg/virt-launcher"
 	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
 	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/network"
 	"kubevirt.io/kubevirt/pkg/watchdog"
@@ -1618,13 +1617,14 @@ type MockGracefulShutdown struct {
 }
 
 func (m *MockGracefulShutdown) TriggerShutdown(vmi *v1.VirtualMachineInstance) {
-	Expect(os.MkdirAll(virtlauncher.GracefulShutdownTriggerDir(m.baseDir), os.ModePerm)).To(Succeed())
+	Expect(os.MkdirAll(filepath.Join(m.baseDir, "graceful-shutdown-trigger"), os.ModePerm)).To(Succeed())
 
 	namespace := precond.MustNotBeEmpty(vmi.GetObjectMeta().GetNamespace())
 	domain := precond.MustNotBeEmpty(vmi.GetObjectMeta().GetName())
-	triggerFile := virtlauncher.GracefulShutdownTriggerFromNamespaceName(m.baseDir, namespace, domain)
-	err := virtlauncher.GracefulShutdownTriggerInitiate(triggerFile)
+	triggerFile := gracefulShutdownTriggerFromNamespaceName(m.baseDir, namespace, domain)
+	f, err := os.Create(triggerFile)
 	Expect(err).NotTo(HaveOccurred())
+	f.Close()
 }
 
 type MockWatchdog struct {

pkg/virt-launcher/BUILD.bazel

@@ -7,11 +7,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/ephemeral-disk-utils:go_default_library",
-        "//pkg/virt-handler/cmd-client:go_default_library",
-        "//pkg/watchdog:go_default_library",
-        "//staging/src/kubevirt.io/client-go/api/v1:go_default_library",
         "//staging/src/kubevirt.io/client-go/log:go_default_library",
-        "//staging/src/kubevirt.io/client-go/precond:go_default_library",
     ],
 )