Skip to content

Commit

Permalink
[test] dump serial console log for debug purposes
Browse files Browse the repository at this point in the history
Functional test
"it should not skip any log line even trying to
flood the serial console for QOSGuaranteed VMs"
is generating many sequential log lines over the serial
console fetching the output from k8s log api
and ensuring that all the expected logs lines are
there and in the correct order.

Looking at periodic jobs results the test
is sometimes (~10%) failing due to duplicated
chars in the logs, something like:
```
logline 0006195 0006195 0006195 0006195 0006195 0006195 0006195 0006195 0006195 0006195 0006195 0006195 0006195 0006195 0006195
loglline 0006196 0006196 0006196 0006196 0006196 0006196 0006196 0006196 0006196 0006196 0006196 0006196 0006196 0006196 0006196
logline 0006197 0006197 0006197 0006197 0006197 0006197 0006197 0006197 0006197 0006197 0006197 0006197 0006197 0006197 0006197
```

In case of a failure,
let's collect the console log from virt-launcher
pod with an out-of-band mechanism
to be able to understand if the unexpected chars
was already there in the libvirt generated files.

Signed-off-by: Simone Tiraboschi <[email protected]>
  • Loading branch information
tiraboschi committed Nov 29, 2023
1 parent 95b564d commit 8d32fa6
Showing 1 changed file with 24 additions and 3 deletions.
27 changes: 24 additions & 3 deletions tests/guestlog/guestlog.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package guestlog
import (
"context"
"fmt"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
Expand Down Expand Up @@ -91,6 +93,8 @@ var _ = Describe("[sig-compute]Guest console log", decorators.SigCompute, func()
})

Context("fetch logs", func() {
var vmi *v1.VirtualMachineInstance

var cirrosLogo = `
____ ____ ____
/ __/ __ ____ ____ / __ \/ __/
Expand All @@ -100,7 +104,7 @@ var _ = Describe("[sig-compute]Guest console log", decorators.SigCompute, func()
`

It("it should fetch logs for a running VM with logs API", func() {
vmi := tests.RunVMIAndExpectLaunch(cirrosVmi, cirrosStartupTimeout)
vmi = tests.RunVMIAndExpectLaunch(cirrosVmi, cirrosStartupTimeout)

By("Finding virt-launcher pod")
virtlauncherPod, err := libvmi.GetPodByVirtualMachineInstance(vmi, testsuite.GetTestNamespace(vmi))
Expand Down Expand Up @@ -149,7 +153,7 @@ var _ = Describe("[sig-compute]Guest console log", decorators.SigCompute, func()
})

It("it should rotate the internal log files", func() {
vmi := tests.RunVMIAndExpectLaunch(cirrosVmi, cirrosStartupTimeout)
vmi = tests.RunVMIAndExpectLaunch(cirrosVmi, cirrosStartupTimeout)

By("Finding virt-launcher pod")
virtlauncherPod, err := libvmi.GetPodByVirtualMachineInstance(vmi, testsuite.GetTestNamespace(vmi))
Expand Down Expand Up @@ -179,7 +183,7 @@ var _ = Describe("[sig-compute]Guest console log", decorators.SigCompute, func()
k8sv1.ResourceMemory: resource.MustParse("256M"),
},
}
vmi := tests.RunVMIAndExpectLaunch(cirrosVmi, cirrosStartupTimeout)
vmi = tests.RunVMIAndExpectLaunch(cirrosVmi, cirrosStartupTimeout)
Expect(vmi.Status.QOSClass).ToNot(BeNil())
Expect(*vmi.Status.QOSClass).To(Equal(k8sv1.PodQOSGuaranteed))

Expand Down Expand Up @@ -219,6 +223,23 @@ var _ = Describe("[sig-compute]Guest console log", decorators.SigCompute, func()
Expect(matchingLines).To(BeNumerically(">", 1000))
})

AfterEach(func() {
if CurrentSpecReport().Failed() {
if vmi != nil {
virtlauncherPod, err := libvmi.GetPodByVirtualMachineInstance(vmi, testsuite.GetTestNamespace(vmi))
if err == nil {
artifactsDir, _ := os.LookupEnv("ARTIFACTS")
outputString, err := exec.ExecuteCommandOnPod(virtClient, virtlauncherPod, "guest-console-log", []string{"/bin/bash", "-c", "/bin/tail -v -n +1 " + fmt.Sprintf("/var/run/kubevirt-private/%v/virt-serial*-log*", vmi.UID)})
if err == nil {
lpath := filepath.Join(artifactsDir, fmt.Sprintf("serial_logs_content_%v.txt", vmi.UID))
_, _ = fmt.Fprintf(GinkgoWriter, "Serial console log failed, serial console logs dump from virt-launcher pod collected at file at %s\n", lpath)
_ = os.WriteFile(lpath, []byte(outputString), 0644)
}
}
}
}
})

})

})
Expand Down

0 comments on commit 8d32fa6

Please sign in to comment.