Skip to content

Commit

Permalink
Merge pull request kubevirt#10110 from tiraboschi/console_log_poc
Browse files Browse the repository at this point in the history
Stream guest serial console logs from a dedicated container
kubevirt-bot authored Oct 2, 2023
2 parents 8e2df41 + fec198b commit 1e7849b
Showing 77 changed files with 2,935 additions and 131 deletions.
11 changes: 11 additions & 0 deletions api/openapi-spec/swagger.json
Original file line number Diff line number Diff line change
@@ -16510,6 +16510,10 @@
"$ref": "#/definitions/v1.Interface"
}
},
"logSerialConsole": {
"description": "Whether to log the auto-attached default serial console or not. Serial console logs will be collect to a file and then streamed from a named `guest-console-log`. Not relevant if autoattachSerialConsole is disabled. Defaults to cluster wide setting on VirtualMachineOptions.",
"type": "boolean"
},
"networkInterfaceMultiqueue": {
"description": "If specified, virtual network interfaces configured with a virtio bus will also enable the vhost multiqueue feature for network devices. The number of queues created depends on additional factors of the VirtualMachineInstance, like the number of guest CPUs.",
"type": "boolean"
@@ -16539,6 +16543,9 @@
"v1.DisableFreePageReporting": {
"type": "object"
},
"v1.DisableSerialConsoleLog": {
"type": "object"
},
"v1.Disk": {
"type": "object",
"required": [
@@ -20567,6 +20574,10 @@
"disableFreePageReporting": {
"description": "DisableFreePageReporting disable the free page reporting of memory balloon device https://libvirt.org/formatdomain.html#memory-balloon-device. This will have effect only if AutoattachMemBalloon is not false and the vmi is not requesting any high performance feature (dedicatedCPU/realtime/hugePages), in which free page reporting is always disabled.",
"$ref": "#/definitions/v1.DisableFreePageReporting"
},
"disableSerialConsoleLog": {
"description": "DisableSerialConsoleLog disables logging the auto-attached default serial console. If not set, serial console logs will be written to a file and then streamed from a container named `guest-console-log`. The value can be individually overridden for each VM, not relevant if AutoattachSerialConsole is disabled.",
"$ref": "#/definitions/v1.DisableSerialConsoleLog"
}
}
},
1 change: 1 addition & 0 deletions cmd/virt-launcher-monitor/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@ go_library(
importpath = "kubevirt.io/kubevirt/cmd/virt-launcher-monitor",
visibility = ["//visibility:private"],
deps = [
"//pkg/util:go_default_library",
"//staging/src/kubevirt.io/client-go/log:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/golang.org/x/sys/unix:go_default_library",
68 changes: 66 additions & 2 deletions cmd/virt-launcher-monitor/virt-launcher-monitor.go
Original file line number Diff line number Diff line change
@@ -38,6 +38,8 @@ import (

"golang.org/x/sys/unix"
"kubevirt.io/client-go/log"

"kubevirt.io/kubevirt/pkg/util"
)

const (
@@ -56,10 +58,59 @@ func cleanupContainerDiskDirectory(ephemeralDiskDir string) {
}
}

func createSerialConsoleTermFile(uid, suffix string) (bool, error) {
// Create a file that it will be removed to quickly signal the
// shutdown to the guest-console-log container in the case the sigterm signal got
// missed and some client process is still connected to the serial console socket
const serialPort = 0
if len(uid) > 0 {
logSigPath := fmt.Sprintf("%s/%s/virt-serial%d-log-sigTerm%s", util.VirtPrivateDir, uid, serialPort, suffix)

if _, err := os.Stat(logSigPath); os.IsNotExist(err) {
file, err := os.Create(logSigPath)
if err != nil {
log.Log.Reason(err).Errorf("could not create up serial console term file: %s", logSigPath)
return false, err
}
if err = file.Close(); err != nil {
log.Log.Reason(err).Errorf("could not create up serial console term file: %s", logSigPath)
return false, err
}
log.Log.V(3).Infof("serial console term file created: %s", logSigPath)
return true, nil
}
}
return false, nil

}

func removeSerialConsoleTermFile(uid string) {
// Delete a file (if there) to quickly signal the shutdown to the guest-console-log container in the case the sigterm signal got
// missed and some client process is still connected to the serial console socket
const serialPort = 0
if len(uid) > 0 {
logSigPath := fmt.Sprintf("%s/%s/virt-serial%d-log-sigTerm", util.VirtPrivateDir, uid, serialPort)

if _, err := os.Stat(logSigPath); err == nil {
rerr := os.Remove(logSigPath)
if rerr != nil {
log.Log.Reason(err).Errorf("could not delete serial console term file: %s", logSigPath)
return
}
log.Log.V(3).Infof("serial console term file deleted: %s", logSigPath)
}
}
// Create a second termination file for the unlikely case where virt-launcher-monitor
// has enough time to create and remove the termination file before virt-tail (asynchronously started)
// notices it.
createSerialConsoleTermFile(uid, "-done")
}

func main() {

containerDiskDir := pflag.String("container-disk-dir", "/var/run/kubevirt/container-disks", "Base directory for container disk data")
keepAfterFailure := pflag.Bool("keep-after-failure", false, "virt-launcher will be kept alive after failure for debugging if set to true")
uid := pflag.String("uid", "", "UID of the VirtualMachineInstance")

// set new default verbosity, was set to 0 by glog
goflag.Set("v", "2")
@@ -79,7 +130,7 @@ func main() {
}
}

exitCode, err := RunAndMonitor(*containerDiskDir)
exitCode, err := RunAndMonitor(*containerDiskDir, *uid)
if *keepAfterFailure && (exitCode != 0 || err != nil) {
log.Log.Infof("keeping virt-launcher container alive since --keep-after-failure is set to true")
<-make(chan struct{})
@@ -95,11 +146,24 @@ func main() {

// RunAndMonitor run virt-launcher process and monitor it to give qemu an extra grace period to properly terminate
// in case of crashes
func RunAndMonitor(containerDiskDir string) (int, error) {
func RunAndMonitor(containerDiskDir, uid string) (int, error) {
defer removeSerialConsoleTermFile(uid)
defer cleanupContainerDiskDirectory(containerDiskDir)
defer terminateIstioProxy()
args := removeArg(os.Args[1:], "--keep-after-failure")

go func() {
created := false
i := 0
for i < 100 && !created {
i = i + 1
created, err := createSerialConsoleTermFile(uid, "")
if err != nil || !created {
time.Sleep(100 * time.Millisecond)
}
}
}()

cmd := exec.Command("/usr/bin/virt-launcher", args...)
cmd.SysProcAttr = &syscall.SysProcAttr{
AmbientCaps: []uintptr{unix.CAP_NET_BIND_SERVICE},
1 change: 1 addition & 0 deletions cmd/virt-launcher/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -62,6 +62,7 @@ pkg_tar(
"//cmd/virt-freezer",
"//cmd/virt-launcher-monitor",
"//cmd/virt-probe",
"//cmd/virt-tail",
],
package_dir = "/usr/bin",
)
5 changes: 3 additions & 2 deletions cmd/virt-launcher/virt-launcher.go
Original file line number Diff line number Diff line change
@@ -374,6 +374,9 @@ func main() {
}
}

// Initialize local and shared directories
initializeDirs(*ephemeralDiskDir, *containerDiskDir, *hotplugDiskDir, *uid)

if *simulateCrash {
panic(fmt.Errorf("Simulated virt-launcher crash"))
}
@@ -387,8 +390,6 @@ func main() {

vmi := v1.NewVMIReferenceWithUUID(*namespace, *name, types.UID(*uid))

// Initialize local and shared directories
initializeDirs(*ephemeralDiskDir, *containerDiskDir, *hotplugDiskDir, *uid)
ephemeralDiskCreator := ephemeraldisk.NewEphemeralDiskCreator(filepath.Join(*ephemeralDiskDir, "disk-data"))
if err := ephemeralDiskCreator.Init(); err != nil {
panic(err)
21 changes: 21 additions & 0 deletions cmd/virt-tail/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")

go_library(
name = "go_default_library",
srcs = ["main.go"],
importpath = "kubevirt.io/kubevirt/cmd/virt-tail",
visibility = ["//visibility:private"],
deps = [
"//staging/src/kubevirt.io/client-go/log:go_default_library",
"//vendor/github.com/fsnotify/fsnotify:go_default_library",
"//vendor/github.com/nxadm/tail:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/golang.org/x/sync/errgroup:go_default_library",
],
)

go_binary(
name = "virt-tail",
embed = [":go_default_library"],
visibility = ["//visibility:public"],
)
Loading

0 comments on commit 1e7849b

Please sign in to comment.